爬虫代码有问题请各位大佬指点

我爬取的网站是https://www.shixi.com/search/index?key=大数据，内容是全部页数的六个信息
这个代码我是照着爬取豆瓣TOP250的那个代码写的，希望有人指点我一下

1. import requests
   
2. import bs4
   
3. import re
   
4. 
   
5. def open_url(url):
   
6.     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
   
7.     res = requests.get(url, headers=headers,verify=False)
   
8.     return res
   
9. 
   
10. def find_occupation(res):
    
11.     soup =bs4.BeautifulSoup(res.text, 'html.parser')
    
12. 
    
13.     #岗位名称
    
14.     post = []
    
15.     targets = soup.find_all("dt", class_="job-name")
    
16.     for each in targets:
    
17.         post.append(each.text)
    
18. 
    
19.     #公司名称
    
20.     company = []
    
21.     targets = soup.find_all("div", class_="comany-info-title")
    
22.     for each in targets:
    
23.         company.append(each.a.text)
    
24. 
    
25.     #工作地点
    
26.     area = []
    
27.     targets = soup.find_all("span", class_="job-address")
    
28.     for each in targets:
    
29.         area.append(each.text)
    
30. 
    
31. 
    
32.     #学历要求
    
33.     education = []
    
34.     targets = soup.find_all("dd", class_="job-des")
    
35.     for each in targets:
    
36.         education.append(each.text)
    
37. 
    
38.     #薪资情况
    
39.     pay = []
    
40.     targets = soup.find_all("div", class_="comany-info-des")
    
41.     for each in targets:
    
42.         pay.append(each.text)
    
43.    
    
44.     #发布时间
    
45.     time = []
    
46.     targets = soup.find_all("span", class_="job-time")
    
47.     for each in targets:
    
48.         try:
    
49.             time.append(each.text.split('\n')[1].strip() + each.text.split('\n')[2].strip())
    
50.         except:
    
51.             continue
    
52. 
    
53.     result = []
    
54.     length = len(post)
    
55.     for i in range(length):
    
56.         result.append(post[i] + company[i] + area[i] + education[i] + pay[i] + time[i] + '\n')
    
57.     return result
    
58. 
    
59. #找出共有多少个页面
    
60. def find_depth(res):
    
61.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
62.     depth = soup.find('li', class_='next').previous_sibling.previous_sibling.text
    
63. 
    
64.     return int(depth)
    
65. 
    
66. def main():
    
67.     host = "https://www.shixi.com/search/index?key=大数据"
    
68.     res = open_url(host)
    
69.     depth = find_depth(res)
    
70. 
    
71.     result = []
    
72.     for i in range(depth):
    
73.         url = host + '&page=' + str(10 * i)
    
74.         res = open_url(url)
    
75.         result.extend(find_occupation(res))
    
76. 
    
77.     with open ("大数据.txt", "w", encoding="utf-8") as f:
    
78.         for each in result:
    
79.             f.write(each)
    
80. 
    
81. 
    
82. if __name__ == "__main__":
    
83.     main()   
    

复制代码

槐月余八

顶一下，有没有大佬帮忙解答一下啊，孩子球球了

槐月余八

suchocolate 发表于 2021-12-19 14:49
find_movies函数是做什么 ？

是我忽略了没改，应给是 find_occuptation(res)
虽然还未能运行，不过谢谢指出

suchocolate

find_movies函数是做什么 ？