.net抓取百度造成。百度局域网异常访问
直接贴代码。
public static HttpWebResponse CreateGetHttpResponse(string url, int? timeout, string userAgent, CookieCollection cookies)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException("url");
}
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.Method = "GET";
request.UserAgent = DefaultUserAgent;
if (!string.IsNullOrEmpty(userAgent))
{
request.UserAgent = userAgent;
}
if (timeout.HasValue)
{
request.Timeout = timeout.Value;
}
if (cookies != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookies);
}
return request.GetResponse() as HttpWebResponse;
}
aTimer = new System.Timers.Timer(86400000);
aTimer.Elapsed += new ElapsedEventHandler(OnTimedEvent);
aTimer.Interval = 3000;
aTimer.Enabled = true;
List<Qy> qys = Get_Coms();//所有法人
List<Qy> qysss = qys.Distinct(c => c.QYID).ToList();//所有企业
List<string> keys = Get_Keys();
//企业
foreach (var item in qysss)
{
foreach (var key in keys)
{
int num = 10;//搜索条数
string url = "http://www.baidu.com/s?wd=" + item.QYName + " " + key + "&rn=" + num + "";//根据企业名称+关键字搜索
Uri urls = new Uri(url);
string html = html = search(url, "gb2312");//gb2312
//string html = HttpWebResponseUtility.GetHttpSourceValue(url);
//string html = GetHTMLTCP(url);
// string html2 = GetHTML(url,"gb2312");
//string html = HttpWebResponseUtility.GetHtmlSource2("http://www.baidu.com");
string html = HttpWebResponseUtility.GetHtml(url, Encoding.UTF8);
//this.webBrowser1.Url = urls;
BaiduSearch baidu = new BaiduSearch();
if (!string.IsNullOrEmpty(html))
{
int count = baidu.GetSearchCount(html);//搜索条数
if (count > 0)
{
List<Keyword> keywords = baidu.GetKeywords(html, item.QYID, item.QYName, "");
//dataGridView1.DataSource = keywords;
}
if (count == 0)
{
var time = DateTime.Now;
}
}
}
}
//企业法人
foreach (var item in qys)
{
foreach (var key in keys)
{
if (!string.IsNullOrEmpty(item.FName))
{
int num = 10;//搜索条数
string url = "http://www.baidu.com/s?wd=" + item.FName + " " + key + "&rn=" + num + "";//根据企业法人名字+关键字搜索
Uri urls = new Uri(url);
string html = search(url, "gb2312");//UTF-8
//this.webBrowser1.Url = urls;
BaiduSearch baidu = new BaiduSearch();
if (!string.IsNullOrEmpty(html))
{
int count = baidu.GetSearchCount(html);//搜索条数
if (count > 0)
{
List<Keyword> keywords = baidu.GetKeywords(html, item.QYID, item.QYName, item.FName);
//dataGridView1.DataSource = keywords;
}
}
}
}
}
}
