对python抓取需要登录网站数据的方法详解
scrapy.FormRequest
login.py
classLoginSpider(scrapy.Spider):
name='login_spider'
start_urls=['http://www.login.com']
defparse(self,response):
return[
scrapy.FormRequest.from_response(
response,
#username和password要根据实际页面的表单的name字段进行修改
formdata={'username':'your_username','password':'your_password'},
callback=self.after_login)]
defafter_login(self,response):
#登录后的代码
pass
selenium登录获取cookie
get_cookie_by_selenium.py
importpickle
importtime
fromseleniumimportwebdriver
defget_cookies():
url='https://www.test.com'
web_driver=webdriver.Chrome()
web_driver.get(url)
username=web_driver.find_element_by_id('login-email')
username.send_keys('username')
password=web_driver.find_element_by_id('login-password')
password.send_keys('password')
login_button=web_driver.find_element_by_id('login-submit')
login_button.click()
time.sleep(3)
cookies=web_driver.get_cookies()
web_driver.close()
returncookies
if__name__=='__main__':
cookies=get_cookies()
pickle.dump(cookies,open('cookies.pkl','wb'))
获取浏览器cookie(以Ubuntu的Firefox为例)
get_cookie_by_firefox.py
importsqlite3
importpickle
defget_cookie_by_firefox():
cookie_path='/home/name/.mozilla/firefox/bqtvfe08.default/cookies.sqlite'
withsqlite3.connect(cookie_path)asconn:
sql='selectname,valuefrommoz_cookieswherebaseDomain="test.com"'
cur=conn.cursor()
cookies=[{'name':name,'value':value}forname,valueincur.execute(sql).fetchall()]
returncookies
if__name__=='__main__':
cookies=get_cookie_from_firefox()
pickle.dump(cookies,open('cookies.pkl','wb'))
scrapy使用获取后的cookie
cookies=pickle.load(open('cookies.pkl','rb'))
yieldscrapy.Request(url,cookies=cookies,callback=self.parse)
requests使用获取后的cookie
cookies=pickle.load(open('cookies.pkl','rb'))
s=requests.Session()
forcookieincookies:
s.cookies.set(cookie['name'],cookie['value'])
selenium使用获取后的cookie
fromseleniumimportwebdriver
cookies=pickle.load(open('cookies.pkl','rb'))
w=webdriver.Chrome()
#直接添加cookie会报错，下面是一种解决方案，可能有更好的
#--start--
w.get('http://www.test.com')
w.delete_all_cookies()
#--end--
forcookieincookies:
w.add_cookie(cookie)
以上这篇对python抓取需要登录网站数据的方法详解就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持毛票票。