爬虫

私はり

我爬取boss直聘，返回的是空列表，不知道怎么回事，还有就是我想要将第一个页面到第十个页面分别写入十个csv文件该怎么写？

1. import urllib.request
   
2. import urllib.parse
   
3. from lxml import etree
   
4. def creat_request(my_page):
   
5.     base_url = 'https://www.zhipin.com/c101270100/?'
   
6.     headers = {
   
7.         'Accept': ' text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
   
8.         # 'Accept-Encoding': ' gzip, deflate, br',
   
9.         'Accept-Language': ' zh,en-US;q=0.9,en;q=0.8,zh-CN;q=0.7',
   
10.         'Connection': ' keep-alive',
    
11.         'Cookie': ' acw_tc=0bcb2f0716324609378892612e49ac132e87df9e2107cbb14b08e84e449455; __g=-; Hm_lvt_194df3105ad7148dcf2b98a91b5e727a=1632460940; lastCity=100010000; __l=l=%2Fwww.zhipin.com%2Fc101270100%2F%3Fka%3Dsel-city-101270100&r=https%3A%2F%2Fcn.bing.com%2F&g=&s=3&friend_source=0&s=3&friend_source=0; __c=1632460940; __a=34040189.1632460940..1632460940.9.1.9.9; Hm_lpvt_194df3105ad7148dcf2b98a91b5e727a=1632461742; __zp_stoken__=f68bdWDVKZBhBNRMPdCExbkphQAJsIzQwI3ROPh9gXUpCAQYhd3ENZnFRIh87IjdMJkx%2BFx06Dy43OjZAYwE9CB9LDXIeIWlAQR5kD1QNLx9LKzFwTQZrYCAHenNLDxEMZHVMP2BODQ1gdEY%3D',
    
12.         'Host': ' www.zhipin.com',
    
13.         'sec-ch-ua': ' "Chromium";v="94", "Google Chrome";v="94", ";Not A Brand";v="99"',
    
14.         'sec-ch-ua-mobile': ' ?0',
    
15.         'sec-ch-ua-platform': ' "Windows"',
    
16.         'Sec-Fetch-Dest': ' document',
    
17.         'Sec-Fetch-Mode': ' navigate',
    
18.         'Sec-Fetch-Site': ' none',
    
19.         'Sec-Fetch-User': ' ?1',
    
20.         'Upgrade-Insecure-Requests': ' 1',
    
21.         'User-Agent': ' Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36',
    
22.     }
    
23.     data = {
    
24.         'page' : my_page,
    
25.         'ka' : 'page-'+ str(my_page),
    
26.     }
    
27.     data = urllib.parse.urlencode(data)
    
28.     url = base_url + data
    
29.     print(url)
    
30.     request = urllib.request.Request(url=url,headers=headers)
    
31.     return request
    
32. 
    
33. def get_content(request):
    
34.     response = urllib.request.urlopen(request)
    
35.     content = response.read().decode('utf-8')
    
36.     return content
    
37. 
    
38. def down_load(content):
    
39.     tree = etree.HTML(content)
    
40.     name_list = tree.xpath('//*[@id="main"]/div/div[2]/ul/li//div/span/a/text()')
    
41.     place_list = tree.xpath('//*[@id="main"]/div/div[2]/ul/li//div/span[2]/span/text()')
    
42.     for i in range(len(name_list)):
    
43.         name = name_list[i]
    
44.         place = place_list[i]
    
45.     print(place_list)
    
46. 
    
47. 
    
48. 
    
49. if __name__ == '__main__':
    
50.     start_page = int(input("请输入起始页码:"))
    
51.     end_page = int(input("请输入结束页码:"))
    
52.     for my_page in range(start_page,end_page+1):
    
53.         request = creat_request(my_page)
    
54.         content = get_content(request)
    
55.         down_load(content)

复制代码

wp231957

cookie 是动态的 有时效性   

私はり

wp231957 发表于 2021-9-30 11:00
cookie 是动态的 有时效性

好的