爬取51job招聘网站内容与网页浏览内容不一致
import requestsimport bs4
import re
def open_url(url):
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language':'en-US,en;q=0.5',
'Accept-Encoding':'gzip',
'DNT':'1',
'Connection':'close'
}
res =requests.get(url,headers=headers)
print(res.text)
with open("save.txt","w",encoding="utf-8") as f:
f.write(res.text)
return res
def main():
#url = "https://search.51job.com/list/000000,000000,0000,00,9,99,+,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare="
url = "https://search.51job.com/list/000000,000000,0000,00,9,99,%25E5%2590%258E%25E7%25AB%25AF%25E5%25BC%2580%25E5%258F%2591,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare="
open_url(url)
if __name__ =='__main__':
main()
爬取下来保存的txt文件里的window.__SEARCH_RESULT__ ={}里的内容在网页端找不到，网页端的东西在txt文件里也找不到，请问各位大佬是什么原因呢 我发现原来是网页版我登陆了账号，把账号注销了就一致了，但是我还是想问一下为什么登入账号后内容会不一致呢？应该采用登录账号后爬取好还是不登陆爬取好？ 937135952 发表于 2021-9-6 16:31
我发现原来是网页版我登陆了账号，把账号注销了就一致了，但是我还是想问一下为什么登入账号后内容会不一致 ...
这要看你想提取啥数据了
很多网站有些数据不登录 无法提取
页:
[1]