方法1 比較推薦
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
/// <summary> /// 用HttpWebRequest取得網頁源碼 /// 對于帶BOM的網頁很有效，不管是什么編碼都能正確識別 /// </summary> /// <param name="url">網頁地址" </param> /// <returns>返回網頁源文件</returns> public static string GetHtmlSource2(string url) { //處理內容 string html = ""; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.Accept = "*/*"; //接受任意文件 request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322)"; // 模擬使用IE在瀏覽 http://www.jfrwli.cn request.AllowAutoRedirect = true;//是否允許302 //request.CookieContainer = new CookieContainer();//cookie容器， request.Referer = url; //當前頁面的引用 HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Stream stream = response.GetResponseStream(); StreamReader reader = new StreamReader(stream, Encoding.Default); html = reader.ReadToEnd(); stream.Close(); return html; } |
方法2
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
using System;using System.Collections.Generic;using System.Linq;using System.Web;using System.IO;using System.Text;using System.Net;namespace MySql{ public class GetHttpData { public static string GetHttpData2(string Url) { string sException = null; string sRslt = null; WebResponse oWebRps = null; WebRequest oWebRqst = WebRequest.Create(Url); oWebRqst.Timeout = 50000; try { oWebRps = oWebRqst.GetResponse(); } catch (WebException e) { sException = e.Message.ToString(); } catch (Exception e) { sException = e.ToString(); } finally { if (oWebRps != null) { StreamReader oStreamRd = new StreamReader(oWebRps.GetResponseStream(), Encoding.GetEncoding("utf-8")); sRslt = oStreamRd.ReadToEnd(); oStreamRd.Close(); oWebRps.Close(); } } return sRslt; } }} |
方法3
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
public static string getHtml(string url, params string [] charSets)//url是要訪問的網站地址，charSet是目標網頁的編碼，如果傳入的是null或者""，那就自動分析網頁的編碼 { try { string charSet = null; if (charSets.Length == 1) { charSet = charSets[0]; } WebClient myWebClient = new WebClient(); //創建WebClient實例myWebClient // 需要注意的： //有的網頁可能下不下來，有種種原因比如需要cookie,編碼問題等等 //這是就要具體問題具體分析比如在頭部加入cookie // webclient.Headers.Add("Cookie", cookie); //這樣可能需要一些重載方法。根據需要寫就可以了 //獲取或設置用于對向 Internet 資源的請求進行身份驗證的網絡憑據。 myWebClient.Credentials = CredentialCache.DefaultCredentials; //如果服務器要驗證用戶名,密碼 //NetworkCredential mycred = new NetworkCredential(struser, strpassword); //myWebClient.Credentials = mycred; //從資源下載數據并返回字節數組。（加@是因為網址中間有"/"符號） byte[] myDataBuffer = myWebClient.DownloadData(url); string strWebData = Encoding.Default.GetString(myDataBuffer); //獲取網頁字符編碼描述信息 Match charSetMatch = Regex.Match(strWebData, "<meta([^<]*)charset=([^<]*)\"", RegexOptions.IgnoreCase | RegexOptions.Multiline); string webCharSet = charSetMatch.Groups[2].Value; if (charSet == null || charSet == "") charSet = webCharSet; if (charSet != null && charSet != "" && Encoding.GetEncoding(charSet) != Encoding.Default) { strWebData = Encoding.GetEncoding(charSet).GetString(myDataBuffer); } else { strWebData = Encoding.GetEncoding("utf-8").GetString(myDataBuffer); } return strWebData; } catch (Exception e) { return ""; } } |
asp.net 獲取網頁源文件的方法
有時候我們需要獲取 網頁源文件，所以用以下這個方法很容易完成任務！
|
1
2
3
4
5
6
7
8
9
|
private string GetStringByUrl(string strUrl) { WebRequest wrt = WebRequest.Create(strUrl); WebResponse wrse = wrt.GetResponse(); Stream strM = wrse.GetResponseStream(); StreamReader SR = new StreamReader(strM, Encoding.GetEncoding("gb2312")); string strallstrm = SR.ReadToEnd(); return strallstrm; } |
只要傳入要下載網頁的地址就OK了！
通過這個方法做個源碼導出:
|
1
2
3
4
5
6
7
8
9
|
private string SaveHTML() { string str = RenderPage("Default2.aspx"); Response.ContentEncoding = System.Text.Encoding.GetEncoding("UTF-8"); //解決中文亂碼 Response.AddHeader("Content-Disposition","attachment;filename=index.html"); //解決中文文件名亂碼 Response.AddHeader("Content-length",str.Length.ToString()); Response.Write(str); Response.End(); ｝ |
以上就是asp.net 抓取網頁源碼的全部代碼了，希望對大家有所幫助。
