基于scrapy实现的简单蜘蛛采集程序
本文实例讲述了基于scrapy实现的简单蜘蛛采集程序。分享给大家供大家参考。具体如下:
#StandardPythonlibraryimports #3rdpartyimports fromscrapy.contrib.spidersimportCrawlSpider,Rule fromscrapy.contrib.linkextractors.sgmlimportSgmlLinkExtractor fromscrapy.selectorimportHtmlXPathSelector #Myimports frompoetry_analysis.itemsimportPoetryAnalysisItem HTML_FILE_NAME=r'.+\.html' classPoetryParser(object): """ Providescommonparsingmethodforpoemsformattedthisonespecificway. """ date_pattern=r'(\d{2}\w{3,9}\d{4})' defparse_poem(self,response): hxs=HtmlXPathSelector(response) item=PoetryAnalysisItem() #Allpoetrytextisinpretags text=hxs.select('//pre/text()').extract() item['text']=''.join(text) item['url']=response.url #head/titlecontainstitle-apoembyauthor title_text=hxs.select('//head/title/text()').extract()[0] item['title'],item['author']=title_text.split('-') item['author']=item['author'].replace('apoemby','') forkeyin['title','author']: item[key]=item[key].strip() item['date']=hxs.select("//p[@class='small']/text()").re(date_pattern) returnitem classPoetrySpider(CrawlSpider,PoetryParser): name='example.com_poetry' allowed_domains=['www.example.com'] root_path='someuser/poetry/' start_urls=['http://www.example.com/someuser/poetry/recent/', 'http://www.example.com/someuser/poetry/less_recent/'] rules=[Rule(SgmlLinkExtractor(allow=[start_urls[0]+HTML_FILE_NAME]), callback='parse_poem'), Rule(SgmlLinkExtractor(allow=[start_urls[1]+HTML_FILE_NAME]), callback='parse_poem')]
希望本文所述对大家的Python程序设计有所帮助。