C#基于正则表达式实现获取网页中所有信息的网页抓取类实例

2024-02-23 21:42:03 48
本文实例讲述了C#基于正则表达式实现获取网页中所有信息的网页抓取类。分享给大家供大家参考，具体如下：
类的代码：
usingSystem;
usingSystem.Data;
usingSystem.Configuration;
usingSystem.Net;
usingSystem.IO;
usingSystem.Text;
usingSystem.Collections.Generic;
usingSystem.Text.RegularExpressions;
usingSystem.Threading;
usingSystem.Web;
usingSystem.Web.UI.MobileControls;
///
///网页类
///
publicclassWebPage
{
#region私有成员
privateUrim_uri;//url
privateListm_links;//此网页上的链接
privatestringm_title;//标题
privatestringm_html;//HTML代码
privatestringm_outstr;//网页可输出的纯文本
privateboolm_good;//网页是否可用
privateintm_pagesize;//网页的大小
privatestaticDictionarywebcookies=newDictionary();//存放所有网页的Cookie
#endregion
#region属性
///
///通过此属性可获得本网页的网址，只读
///
publicstringURL
{
get
{
returnm_uri.AbsoluteUri;
}
}
///
///通过此属性可获得本网页的标题，只读
///
publicstringTitle
{
get
{
if(m_title=="")
{
Regexreg=newRegex(@"(?m)]*>(?(?:\w|\W)*?)</title[^>]*>",RegexOptions.Multiline|RegexOptions.IgnoreCase);
Matchmc=reg.Match(m_html);
if(mc.Success)
m_title=mc.Groups["title"].Value.Trim();
}
returnm_title;
}
}
publicstringM_html
{
get
{
if(m_html==null)
{
m_html="";
}
returnm_html;
}
}
///<summary>
///此属性获得本网页的所有链接信息，只读
///</summary>
publicList<Link>Links
{
get
{
if(m_links.Count==0)getLinks();
returnm_links;
}
}
///<summary>
///此属性返回本网页的全部纯文本信息，只读
///</summary>
publicstringContext
{
get
{
if(m_outstr=="")getContext(Int16.MaxValue);
returnm_outstr;
}
}
///<summary>
///此属性获得本网页的大小
///</summary>
publicintPageSize
{
get
{
returnm_pagesize;
}
}
///<summary>
///此属性获得本网页的所有站内链接
///</summary>
publicList<Link>InsiteLinks
{
get
{
returngetSpecialLinksByUrl("^http://"+m_uri.Host,Int16.MaxValue);
}
}
///<summary>
///此属性表示本网页是否可用
///</summary>
publicboolIsGood
{
get
{
returnm_good;
}
}
///<summary>
///此属性表示网页的所在的网站
///</summary>
publicstringHost
{
get
{
returnm_uri.Host;
}
}
#endregion
///<summary>
///从HTML代码中分析出链接信息
///</summary>
///<returns>List<Link></returns>
privateList<Link>getLinks()
{
if(m_links.Count==0)
{
Regex[]regex=newRegex[2];
regex[0]=newRegex(@"<a\shref\s*=""(?<URL>[^""]*).*?>(?<title>[^<]*)</a>",RegexOptions.IgnoreCase|RegexOptions.Singleline);
regex[1]=newRegex("<[i]*frame[^><]+src=(\"|')?(?<url>([^>\"'\\s)])+)(\"|')?[^>]*>",RegexOptions.IgnoreCase);
for(inti=0;i<2;i++)
{
Matchmatch=regex[i].Match(m_html);
while(match.Success)
{
try
{
stringurl=HttpUtility.UrlDecode(newUri(m_uri,match.Groups["URL"].Value).AbsoluteUri);
stringtext="";
if(i==0)text=newRegex("(<[^>]+>)|(\\s)|()|&|\"",RegexOptions.Multiline|RegexOptions.IgnoreCase).Replace(match.Groups["text"].Value,"");
Linklink=newLink();
link.Text=text;
link.NavigateUrl=url;
m_links.Add(link);
}
catch(Exceptionex){Console.WriteLine(ex.Message);};
match=match.NextMatch();
}
}
}
returnm_links;
}
///<summary>
///此私有方法从一段HTML文本中提取出一定字数的纯文本
///</summary>
///<paramname="instr">HTML代码</param>
///<paramname="firstN">提取从头数多少个字</param>
///<paramname="withLink">是否要链接里面的字</param>
///<returns>纯文本</returns>
privatestringgetFirstNchar(stringinstr,intfirstN,boolwithLink)
{
if(m_outstr=="")
{
m_outstr=instr.Clone()asstring;
m_outstr=newRegex(@"(?m)<script[^>]*>(\w|\W)*?</script[^>]*>",RegexOptions.Multiline|RegexOptions.IgnoreCase).Replace(m_outstr,"");
m_outstr=newRegex(@"(?m)<style[^>]*>(\w|\W)*?</style[^>]*>",RegexOptions.Multiline|RegexOptions.IgnoreCase).Replace(m_outstr,"");
m_outstr=newRegex(@"(?m)<select[^>]*>(\w|\W)*?</select[^>]*>",RegexOptions.Multiline|RegexOptions.IgnoreCase).Replace(m_outstr,"");
if(!withLink)m_outstr=newRegex(@"(?m)<a[^>]*>(\w|\W)*?</a[^>]*>",RegexOptions.Multiline|RegexOptions.IgnoreCase).Replace(m_outstr,"");
RegexobjReg=newSystem.Text.RegularExpressions.Regex("(<[^>]+?>)|",RegexOptions.Multiline|RegexOptions.IgnoreCase);
m_outstr=objReg.Replace(m_outstr,"");
RegexobjReg2=newSystem.Text.RegularExpressions.Regex("(\\s)+",RegexOptions.Multiline|RegexOptions.IgnoreCase);
m_outstr=objReg2.Replace(m_outstr,"");
}
returnm_outstr.Length>firstN?m_outstr.Substring(0,firstN):m_outstr;
}
#region公有文法
///<summary>
///此公有方法提取网页中一定字数的纯文本，包括链接文字
///</summary>
///<paramname="firstN">字数</param>
///<returns></returns>
publicstringgetContext(intfirstN)
{
returngetFirstNchar(m_html,firstN,true);
}
///<summary>
///此公有方法从本网页的链接中提取一定数量的链接，该链接的URL满足某正则式
///</summary>
///<paramname="pattern">正则式</param>
///<paramname="count">返回的链接的个数</param>
///<returns>List<Link></returns>
publicList<Link>getSpecialLinksByUrl(stringpattern,intcount)
{
if(m_links.Count==0)getLinks();
List<Link>SpecialLinks=newList<Link>();
List<Link>.Enumeratori;
i=m_links.GetEnumerator();
intcnt=0;
while(i.MoveNext()&&cnt<count)
{
if(newRegex(pattern,RegexOptions.Multiline|RegexOptions.IgnoreCase).Match(i.Current.NavigateUrl).Success)
{
SpecialLinks.Add(i.Current);
cnt++;
}
}
returnSpecialLinks;
}
///<summary>
///此公有方法从本网页的链接中提取一定数量的链接，该链接的文字满足某正则式
///</summary>
///<paramname="pattern">正则式</param>
///<paramname="count">返回的链接的个数</param>
///<returns>List<Link></returns>
publicList<Link>getSpecialLinksByText(stringpattern,intcount)
{
if(m_links.Count==0)getLinks();
List<Link>SpecialLinks=newList<Link>();
List<Link>.Enumeratori;
i=m_links.GetEnumerator();
intcnt=0;
while(i.MoveNext()&&cnt<count)
{
if(newRegex(pattern,RegexOptions.Multiline|RegexOptions.IgnoreCase).Match(i.Current.Text).Success)
{
SpecialLinks.Add(i.Current);
cnt++;
}
}
returnSpecialLinks;
}
///<summary>
///这公有方法提取本网页的纯文本中满足某正则式的文字by何问起
///</summary>
///<paramname="pattern">正则式</param>
///<returns>返回文字</returns>
publicstringgetSpecialWords(stringpattern)
{
if(m_outstr=="")getContext(Int16.MaxValue);
Regexregex=newRegex(pattern,RegexOptions.Multiline|RegexOptions.IgnoreCase);
Matchmc=regex.Match(m_outstr);
if(mc.Success)
returnmc.Groups[1].Value;
returnstring.Empty;
}
#endregion
#region构造函数
privatevoidInit(string_url)
{
try
{
m_uri=newUri(_url);
m_links=newList<Link>();
m_html="";
m_outstr="";
m_title="";
m_good=true;
if(_url.EndsWith(".rar")||_url.EndsWith(".dat")||_url.EndsWith(".msi"))
{
m_good=false;
return;
}
HttpWebRequestrqst=(HttpWebRequest)WebRequest.Create(m_uri);
rqst.AllowAutoRedirect=true;
rqst.MaximumAutomaticRedirections=3;
rqst.UserAgent="Mozilla/4.0(compatible;MSIE5.01;WindowsNT5.0)";
rqst.KeepAlive=true;
rqst.Timeout=10000;
lock(WebPage.webcookies)
{
if(WebPage.webcookies.ContainsKey(m_uri.Host))
rqst.CookieContainer=WebPage.webcookies[m_uri.Host];
else
{
CookieContainercc=newCookieContainer();
WebPage.webcookies[m_uri.Host]=cc;
rqst.CookieContainer=cc;
}
}
HttpWebResponsersps=(HttpWebResponse)rqst.GetResponse();
Streamsm=rsps.GetResponseStream();
if(!rsps.ContentType.ToLower().StartsWith("text/")||rsps.ContentLength>1<<22)
{
rsps.Close();
m_good=false;
return;
}
Encodingcding=System.Text.Encoding.Default;
stringcontenttype=rsps.ContentType.ToLower();
intix=contenttype.IndexOf("charset=");
if(ix!=-1)
{
try
{
cding=System.Text.Encoding.GetEncoding(rsps.ContentType.Substring(ix+"charset".Length+1));
}
catch
{
cding=Encoding.Default;
}
//该处视情况而定有的需要解码
//m_html=HttpUtility.HtmlDecode(newStreamReader(sm,cding).ReadToEnd());
m_html=newStreamReader(sm,cding).ReadToEnd();
}
else
{
//该处视情况而定有的需要解码
//m_html=HttpUtility.HtmlDecode(newStreamReader(sm,cding).ReadToEnd());
m_html=newStreamReader(sm,cding).ReadToEnd();
Regexregex=newRegex("charset=(?<cding>[^=]+)?\"",RegexOptions.IgnoreCase);
stringstrcding=regex.Match(m_html).Groups["cding"].Value;
try
{
cding=Encoding.GetEncoding(strcding);
}
catch
{
cding=Encoding.Default;
}
byte[]bytes=Encoding.Default.GetBytes(m_html.ToCharArray());
m_html=cding.GetString(bytes);
if(m_html.Split('?').Length>100)
{
m_html=Encoding.Default.GetString(bytes);
}
}
m_pagesize=m_html.Length;
m_uri=rsps.ResponseUri;
rsps.Close();
}
catch(Exceptionex)
{
}
}
publicWebPage(string_url)
{
stringuurl="";
try
{
uurl=Uri.UnescapeDataString(_url);
_url=uurl;
}
catch{};
Init(_url);
}
#endregion
}

</pre>
<p>调用：</p>
<pre>
WebPagewebInfo=newWebPage("http://hovertree.net/");
webInfo.Context;//不包含html标签的所有内容
webInfo.M_html;//包含html标签的内容by何问起

</pre>
<p><strong>PS：这里再为大家提供2款非常方便的正则表达式工具供大家参考使用：</strong></p>
<p><strong>JavaScript正则表达式在线测试工具：<br/>
</strong>http://tools.jb51.net/regex/javascript</p>
<p><strong>正则表达式在线生成工具：<br/>
</strong>http://tools.jb51.net/regex/create_reg</p>
<p>更多关于C#相关内容感兴趣的读者可查看本站专题：《C#正则表达式用法总结》、《C#编码操作技巧总结》、《C#中XML文件操作技巧汇总》、《C#常见控件用法教程》、《WinForm控件用法总结》、《C#数据结构与算法教程》、《C#面向对象程序设计入门教程》及《C#程序设计之线程使用技巧总结》</p>
<p>希望本文所述对大家C#程序设计有所帮助。</p></section>
                </div>
            </div>
            <!--文章阅读上下按钮-->  
            <nav>
                                    <span class="meta-nav"><a href="/article/237038.html" title="Ajax实现动态加载数据" se_prerender_url="complete">上一篇<br>Ajax实现动态加载数据</a></span>
                
                                    <span class="meta-nav"><a href="/article/237035.html" title="Android sdutio配置Zxing进行扫码功能的实现方法" se_prerender_url="complete">下一篇<br>Android sdutio配置Zxing进行扫码功能的实现方法</a></span>
                            <div class="clear"></div>
            </nav>
            <!-- 文章推荐 -->
            <div class="article-recommend">         
                <h3>热门推荐</h3> 
                                        <div class="title"><span class="s-1">1</span> <a href="/article/10835.html" title="毛坯房验收经验和常识 看了之后再验房心里有底">毛坯房验收经验和常识 看了之后再验房心里有底</a></div>  
                                        <div class="title"><span class="s-2">2</span> <a href="/article/10697.html" title="二手房收房如何交接 二手房收房注意问题">二手房收房如何交接 二手房收房注意问题</a></div>  
                                        <div class="title"><span class="s-3">3</span> <a href="/article/10605.html" title="专业验收毛坯房的价格 商品房验收合格的标准">专业验收毛坯房的价格 商品房验收合格的标准</a></div>  
                                        <div class="title"><span class="s-4">4</span> <a href="/article/10568.html" title="精装房怎么验收 精装房请验房师有用吗">精装房怎么验收 精装房请验房师有用吗</a></div>  
                                        <div class="title"><span class="s-5">5</span> <a href="/article/10567.html" title="一般要到哪里找验房师 验房师有哪些作用呢">一般要到哪里找验房师 验房师有哪些作用呢</a></div>  
                                        <div class="title"><span class="s-6">6</span> <a href="/article/10566.html" title="请人验房一般是多少钱 验房师费用是多少">请人验房一般是多少钱 验房师费用是多少</a></div>  
                                        <div class="title"><span class="s-7">7</span> <a href="/article/10553.html" title="怎样测量房子面积 建筑面积和使用面积怎么算">怎样测量房子面积 建筑面积和使用面积怎么算</a></div>  
                                        <div class="title"><span class="s-8">8</span> <a href="/article/10550.html" title="收房需要注意什么 仔细检查不松懈">收房需要注意什么 仔细检查不松懈</a></div>  
                                        <div class="title"><span class="s-9">9</span> <a href="/article/10548.html" title="收房时三书一证一表是什么 主要作用介绍">收房时三书一证一表是什么 主要作用介绍</a></div>  
                                        <div class="title"><span class="s-10">10</span> <a href="/article/10547.html" title="交房时交房税费有哪些 本文为你一一讲解">交房时交房税费有哪些 本文为你一一讲解</a></div>  
                                        <div class="title"><span class="s-11">11</span> <a href="/article/10526.html" title="验房都需要验什么 要做哪些准备呢">验房都需要验什么 要做哪些准备呢</a></div>  
                                        <div class="title"><span class="s-12">12</span> <a href="/article/10525.html" title="毛坯房验房师有必要请吗 毛坯房装修完如何验收">毛坯房验房师有必要请吗 毛坯房装修完如何验收</a></div>  
                                        <div class="title"><span class="s-13">13</span> <a href="/article/7820.html" title="地下室防水工程质量验收规范详解">地下室防水工程质量验收规范详解</a></div>  
                                        <div class="title"><span class="s-14">14</span> <a href="/article/4947.html" title="水性涂料、油性涂料区别介绍">水性涂料、油性涂料区别介绍</a></div>  
                                        <div class="title"><span class="s-15">15</span> <a href="/article/4517.html" title="零基础布艺DIY工坊 教你做超萌猫头鹰钥匙包">零基础布艺DIY工坊 教你做超萌猫头鹰钥匙包</a></div>  
                                        <div class="title"><span class="s-16">16</span> <a href="/article/4438.html" title="三棵树漆怎么样？三棵树漆官方网站">三棵树漆怎么样？三棵树漆官方网站</a></div>  
                                        <div class="title"><span class="s-17">17</span> <a href="/article/4258.html" title="家庭“装修套餐”中猫腻你知道吗？">家庭“装修套餐”中猫腻你知道吗？</a></div>  
                                        <div class="title"><span class="s-18">18</span> <a href="/article/4051.html" title="小空间大浴望 卫浴间装修巧支招">小空间大浴望 卫浴间装修巧支招</a></div>  
                     
            </div>
            <div class="add-gg">
                <!-- 文章内页广告 -->
                <ins class="adsbygoogle"
                     style="display:block"
                     data-ad-client="ca-pub-4387516810452807"
                     data-ad-slot="9131157744"
                     data-ad-format="auto"
                     data-full-width-responsive="true"></ins>
                <script>
                     (adsbygoogle = window.adsbygoogle || []).push({});
                </script>
            </div>
        </div>
        <!--侧边栏-->
        <div id="sidebar">
            <li class="sidebar-box">
                <h3 class="widget-title">随机推荐</h3>
                <ul>
                                            <li>
                            <a href="/article/99040.html" title="脚为什么会发麻">脚为什么会发麻</a>
                        </li>
                                            <li>
                            <a href="/article/99041.html" title="翔组词有哪些词语">翔组词有哪些词语</a>
                        </li>
                                            <li>
                            <a href="/article/99042.html" title="别人的车堵住了怎么办">别人的车堵住了怎么办</a>
                        </li>
                                            <li>
                            <a href="/article/99043.html" title="八珍颗粒一般要吃多久">八珍颗粒一般要吃多久</a>
                        </li>
                                            <li>
                            <a href="/article/99044.html" title="脾组词有哪些">脾组词有哪些</a>
                        </li>
                                            <li>
                            <a href="/article/99045.html" title="考研圣经和张剑哪个好">考研圣经和张剑哪个好</a>
                        </li>
                                            <li>
                            <a href="/article/99046.html" title="形容注意力集中的四字词语">形容注意力集中的四字词语</a>
                        </li>
                                            <li>
                            <a href="/article/99047.html" title="宝宝无症状发烧原因有哪些">宝宝无症状发烧原因有哪些</a>
                        </li>
                                            <li>
                            <a href="/article/99048.html" title="早孕症状都有哪些">早孕症状都有哪些</a>
                        </li>
                                            <li>
                            <a href="/article/99049.html" title="哪个加速器好用而且免费">哪个加速器好用而且免费</a>
                        </li>
                                            <li>
                            <a href="/article/99050.html" title="德力西是哪个国家的品牌">德力西是哪个国家的品牌</a>
                        </li>
                                            <li>
                            <a href="/article/99052.html" title="地球和火星哪个大">地球和火星哪个大</a>
                        </li>
                                            <li>
                            <a href="/article/99053.html" title="太平洋保险和人寿保险哪个好">太平洋保险和人寿保险哪个好</a>
                        </li>
                                            <li>
                            <a href="/article/99054.html" title="伊丽莎白雅顿是哪个国家的">伊丽莎白雅顿是哪个国家的</a>
                        </li>
                                            <li>
                            <a href="/article/99055.html" title="qq抽礼物在哪里找">qq抽礼物在哪里找</a>
                        </li>
                                            <li>
                            <a href="/article/99056.html" title="腾冲泡温泉哪里好">腾冲泡温泉哪里好</a>
                        </li>
                                    </ul>
            </li>
            <li class="sidebar-box">

                 <!-- google文章侧边 -->
                <ins class="adsbygoogle"
                     style="display:block"
                     data-ad-client="ca-pub-4387516810452807"
                     data-ad-slot="8604338277"
                     data-ad-format="auto"
                     data-full-width-responsive="true"></ins>
                <script>
                     (adsbygoogle = window.adsbygoogle || []).push({});
                </script>
            </li>
        </div>
    </div>
</div>

<ul class="suspend">
    <li class="back-top" onclick="backTop()">
        <i class="fa fa-chevron-up"></i>
        <span class="more">返回顶部</span>
    </li>
    <li>
        <a href="http://wpa.qq.com/msgrd?v=3&uin=514930285&site=qq&menu=yes">
            <i class="fa fa-qq"></i>
            <span class="more">514930285</span>
        </a>
    </li>
    <li>
        <a href="mailto:czq8825@qq.com">
            <i class="fa fa-envelope"></i>
            <span class="more">czq8825@qq.com</span>
        </a>
    </li>
<!--     <li>
        <i class="fa fa-weixin"></i>
        <span class="more weixin"><img src="http://www.haoziyuan.cc/Uploads/20210427/6087c019e1f1f.jpg" alt="微信二维码"></span>
    </li> -->
</ul>

<footer class="footer clearfix">
    <p>免责声明：网站资源来源于网络，如有侵权，请及时联系删除。</p>
    <p>Copyright © 2024 <a href="/">好资源导航网</a>. All Rights Reserved.</p>
    <p>
        <a href="https://beian.miit.gov.cn">蜀ICP备2021004611号-4</a>
        <a href="/sitemap.html">网站地图</a>
        <script>
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?78e2f1027b3a17750580941ea594d61e";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script>    </p>
</footer>
<script src="/Public/Home/js/main.js"></script>
<script type="text/javascript" src="/Public/Home/js/z_stat.js"></script>
</body>
</html>