Python 微信爬虫完整实例【单线程与多线程】

2023-08-20 10:15:04 67

本文实例讲述了Python实现的微信爬虫。分享给大家供大家参考，具体如下：

单线程版：

importurllib.request
importurllib.parse
importurllib.error
importre,time
headers=("User-Agent",
"Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/60.0.3107.4Safari/537.36")
operner=urllib.request.build_opener()
operner.addheaders=[headers]
urllib.request.install_opener(operner)
list_url=[]
###使用代理获取网页url内容
defuse_proxy(url):
try:
#proxy=urllib.request.ProxyHandler({'http':proxy_addr})##使用代理版
#operner=urllib.request.build_opener()
#urllib.request.install_opener(operner)
headers=("User-Agent",
"Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/60.0.3107.4Safari/537.36")
operner=urllib.request.build_opener()
operner.addheaders=[headers]
urllib.request.install_opener(operner)
data=urllib.request.urlopen(url).read().decode('utf-8')
#print(data)
returndata
excepturllib.error.URLErrorase:
ifhasattr(e,"code"):
print(e.code)
elifhasattr(e,"reason"):
print(e.reason)
exceptExceptionase:
print("exception"+str(e))
time.sleep(1)
##获取要爬取的url
defget_url(key,pagestart,pageend):
try:
keycode=urllib.parse.quote(key)
forpageinrange(pagestart,pageend+1):
url="http://weixin.sogou.com/weixin?query=%s&_sug_type_=&s_from=input&_sug_=n&type=%d&page=1&ie=utf8"%(
keycode,page)
data1=use_proxy(url)
#print("data1的内容是",data1)
listurl_pattern='.*?("http://.*?)'
result=re.compile(listurl_pattern,re.S).findall(data1)
foriinrange(len(result)):
res=result[i].replace("amp;","").split("")[0].replace("\"","")
list_url.append(res)
#print(list_url)
returnlist_url
excepturllib.error.URLErrorase:
ifhasattr(e,"code"):
print(e.code)
elifhasattr(e,"reason"):
print(e.reason)
exceptExceptionase:
print("exception:",e)
##通过获取的url爬行内容数据并处理
defget_url_content(list_url):
fh1=open("D:\\python-script\\1.html",'wb')
html1='''\n\n\n\n微信文章\n'''
fh1.write(html1.encode("utf-8"))
fh1.close()
fh=open("D:\\python-script\\1.html",'ab')
forurlinlist_url:
data_content=use_proxy(url)
#print(data_content)
#sys.exit()
title_pattern='.*?'
result_title=re.compile(title_pattern,re.S).findall(data_content)
##标题(str)
res_title=result_title[0].replace("","").replace("",
"").strip()
content_pattern='id="js_content">(.*?)'
content=re.compile(content_pattern,re.S).findall(data_content)
try:
fh.write(res_title.encode("utf-8"))
foriincontent:
fh.write(i.strip().encode("utf-8"))
exceptUnicodeEncodeErrorase:
continue
fh.write("".encode("utf-8"))
if__name__=='__main__':
pagestart=1
pageend=2
key="人工智能"
get_url(key,pagestart,pageend)
get_url_content(list_url)

多线程版：

importurllib.request
importurllib.parse
importurllib.error
importre,time
importqueue
importthreading
headers=("User-Agent","Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/60.0.3107.4Safari/537.36")
operner=urllib.request.build_opener()
operner.addheaders=[headers]
urllib.request.install_opener(operner)
urlque=queue.Queue()
list_url=[]
###使用代理获取网页url内容
defuse_proxy(url):
try:
#proxy=urllib.request.ProxyHandler({'http':proxy_addr})
#operner=urllib.request.build_opener()
#urllib.request.install_opener(operner)
headers=("User-Agent",
"Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/60.0.3107.4Safari/537.36")
operner=urllib.request.build_opener()
operner.addheaders=[headers]
urllib.request.install_opener(operner)
data=urllib.request.urlopen(url).read().decode('utf-8')
#print(data)
returndata
excepturllib.error.URLErrorase:
ifhasattr(e,"code"):
print(e.code)
elifhasattr(e,"reason"):
print(e.reason)
exceptExceptionase:
print("exception"+str(e))
time.sleep(1)
###获取文章的url连接，并将连接加入到队列
classget_url(threading.Thread):
def__init__(self,key,pagestart,pageend,urlque):
threading.Thread.__init__(self)
self.pagestart=pagestart
self.pageend=pageend
self.key=key
self.urlque=urlque
defrun(self):
try:
keycode=urllib.parse.quote(self.key)
forpageinrange(self.pagestart,self.pageend+1):
url="http://weixin.sogou.com/weixin?query=%s&_sug_type_=&s_from=input&_sug_=n&type=%d&page=1&ie=utf8"%(keycode,page)
data=use_proxy(url)
print("data1的内容是",data)
listurl_pattern='.*?("http://.*?)'
result=re.compile(listurl_pattern,re.S).findall(data)
print(result)
iflen(result)==0:
print("没有可用的url")
sys.exit()
foriinrange(len(result)):
res=result[i].replace("amp;","").split("")[0].replace("\"","")
#list_url.append(res)#加入列表
self.urlque.put(res)##加入队列
self.urlque.task_done()
#returnlist_url
excepturllib.error.URLErrorase:
ifhasattr(e,"code"):
print(e.code)
elifhasattr(e,"reason"):
print(e.reason)
exceptExceptionase:
print("exception:",e)
##根据url获取文章内容
classget_url_content(threading.Thread):
def__init__(self,urlque):
threading.Thread.__init__(self)
self.urlque=urlque
defrun(self):
fh1=open("D:\\python-script\\1.html",'wb')
html1='''\n\n\n\n微信文章\n'''
fh1.write(html1.encode("utf-8"))
fh1.close()
fh=open("D:\\python-script\\1.html",'ab')
whileTrue:
try:
url=self.urlque.get()
data_content=use_proxy(url)
title_pattern='.*?'
result_title=re.compile(title_pattern,re.S).findall(data_content)
##标题
res_title=result_title[0].replace("","").replace("","").strip()
content_pattern='id="js_content">(.*?)'
content=re.compile(content_pattern,re.S).findall(data_content)
#c=''
#foriincontent:
###内容
#c_content=i.replace(c,"").replace("
","").replace("","")
fh.write(res_title.encode("utf-8"))
foriincontent:
fh.write(i.strip().encode("utf-8"))
exceptUnicodeEncodeErrorase:
continue
fh.close()
classcontrl(threading.Thread):
def__init__(self,urlqueue):
threading.Thread.__init__(self)
self.urlqueue=urlqueue
whileTrue:
print("程序正在执行")
ifself.urlqueue.empty():
time.sleep(3)
print("程序执行完毕")
exit()
if__name__=='__main__':
pagestart=1
pageend=2
key="人工智能"
get_url=get_url(key,pagestart,pageend,urlque)
get_url.start()
get_content=get_url_content(urlque)
get_content.start()
cntrol=contrl(urlque)
cntrol.start()

更多关于Python相关内容可查看本站专题：《PythonSocket编程技巧总结》、《Python正则表达式用法总结》、《Python数据结构与算法教程》、《Python函数使用技巧总结》、《Python字符串操作技巧汇总》、《Python入门与进阶经典教程》及《Python文件与目录操作技巧汇总》

希望本文所述对大家Python程序设计有所帮助。

Python 微信爬虫完整实例【单线程与多线程】

.?("http://.?)

.?("http://.?)

随机推荐

Python 微信爬虫完整实例【单线程与多线程】

.*?("http://.*?)

.*?("http://.*?)

热门推荐

随机推荐

.?("http://.?)

.?("http://.?)