[已解决]爬豆瓣TOP250失败

changmind

这个是原代码，运行后没有报错，可是没有文件出来，不知道哪里错了。

1. import requests
   
2. import bs4
   
3. import random
   
4. import openpyxl
   
5. 
   
6. def open_url(url):
   
7.     '''
   
8.     ip_poor = ['61.19.145.66:8080','105.27.237.27:80','110.36.239.234:8080'\
   
9.            '110.39.187.50:49850','159.192.141.89:8080','162.243.244.206:80'\
   
10.            '184.149.34.86:51529','181.118.167.104:80','3.6.99.110:80']
    
11.     proxies = {'http':random.choice(ip_poor)}
    
12.     '''        
    
13.     headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}
    
14.    
    
15.     #html = request.get(url=url, headers=headers, proxies=proxies)
    
16.     html = request.get(url,headers=headers)
    
17.    
    
18.     return html
    
19. 
    
20. def find_info(html):
    
21.     content = bs4.BeautifulSoup(html.text,'html.parser')
    
22. 
    
23.     #电影名
    
24.     movie_name = []
    
25.     target_name = content.find_all('div',class_='hd')
    
26.     for each_name in target_name:
    
27.         movie_name.append(each_name.a.span.text)
    
28. 
    
29.     #评分
    
30.     movie_score = []
    
31.     target_score = content.find_all('span',class_='rating5-t')
    
32.     for each_score in target_score:
    
33.         movie_score.append(each_score.text)
    
34. 
    
35.     #信息
    
36.     movie_info = []
    
37.     target_info = content.find_all('div',class_='bd')
    
38.     for each_info in target_score:
    
39.         try:
    
40.             movie_info.append(each_info.p.text.split('\n')[1].strip() + each_info.p.text.split('\n')[2].strip())
    
41.         except:
    
42.             continue
    
43. 
    
44.     content = []
    
45.     for i in range(len(movie_name)):
    
46.         result.append(movie_name[i] + movie_score[i] + movie_info[i])
    
47. 
    
48.     return contenr
    
49. 
    
50. def next_page(html):
    
51.     target = bs4.BeautifulSoup(html.text,'html.parser')
    
52.     next_path = target.find_all('span',class_='next').previous_sibling.previous_sibling.text
    
53.     return int(next_path)
    
54. 
    
55. def save_excel(content):
    
56.     wb = openpyxl.Workbook()
    
57.     ws = wb.active
    
58.     ws['A1'] = '电影名'
    
59.     ws['B1'] = '评分'
    
60.     ws['C1'] = '电影信息'
    
61.     for each in content:
    
62.         ws.append(each)
    
63.     wb.save(r'D:\Desktop\豆瓣TOP250电影.xlsx')
    
64. 
    
65. def main():
    
66.     url = 'https://movie.douban.com/top250'
    
67.     html = open_url(url)
    
68.     next_path = next_path(html)
    
69. 
    
70.     content = []
    
71.     for i in range(next_path):
    
72.         url_next = url + '/?start=' + str(25 * i)
    
73.         html = open_url(url_next)
    
74.         content.extend(find_info(html))
    
75. 
    
76.     save_excel(content)
    
77. 
    
78. if __name__ == 'main':
    
79.     main()
    

复制代码

最佳答案

月排行榜 / 总排行榜

Twilight6

2020-7-2 23:30:10

第16行使用 requests 模块少了个 s

错误：html = request.get(url,headers=headers)
正确：html = requests.get(url,headers=headers)

第 44 行~ 46 行，result 不知从何而来，应该是你不小心写成了 content = []
错误：

1. content = []
   
2.     for i in range(len(movie_name)):
   
3.         result.append(movie_name[i] + movie_score[i] + movie_info[i])
   
4. return contenr

复制代码

正确：

1. result = []
   
2.     for i in range(len(movie_name)):
   
3.         result.append(movie_name[i] + movie_score[i] + movie_info[i])
   
4. return result

复制代码

第 68 行，next_page 写成了 next_path：

错误：next_path = next_path(html)
正确：next_path = next_page(html)

暂时就找到这么多错误，是否运行成功我还没看

跳转到最佳答案楼层

Twilight6

第16行使用 requests 模块少了个 s

错误：html = request.get(url,headers=headers)
正确：html = requests.get(url,headers=headers)

第 44 行~ 46 行，result 不知从何而来，应该是你不小心写成了 content = []
错误：

1. content = []
   
2.     for i in range(len(movie_name)):
   
3.         result.append(movie_name[i] + movie_score[i] + movie_info[i])
   
4. return contenr

复制代码

正确：

1. result = []
   
2.     for i in range(len(movie_name)):
   
3.         result.append(movie_name[i] + movie_score[i] + movie_info[i])
   
4. return result

复制代码

第 68 行，next_page 写成了 next_path：

错误：next_path = next_path(html)
正确：next_path = next_page(html)

暂时就找到这么多错误，是否运行成功我还没看

changmind

Twilight6 发表于 2020-7-2 23:30
第16行使用 requests 模块少了个 s

是我太粗心了，看来要细心一点才行

changmind

Twilight6 发表于 2020-7-2 23:30
第16行使用 requests 模块少了个 s

虽然错误排掉了，可是还是运行失败

Twilight6

changmind 发表于 2020-7-2 23:48
虽然错误排掉了，可是还是运行失败

之前的存稿，也是看着视频和小甲鱼打的，你拿去对照下吧：

1. import requests
   
2. import bs4
   
3. import openpyxl
   
4. 
   
5. 
   
6. def open_url(url):
   
7. 
   
8.     headers = {
   
9.         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
   
10. 
    
11.     res = requests.get(url, headers=headers)
    
12. 
    
13.     return res
    
14. 
    
15. 
    
16. def find_movies(res):
    
17.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
18. 
    
19.     # 电影名
    
20.     movies = []
    
21.     targets = soup.find_all("div", class_="hd")
    
22.     for each in targets:
    
23.         movies.append(each.a.span.text)
    
24. 
    
25.     # 评分
    
26.     ranks = []
    
27.     targets = soup.find_all("span", class_="rating_num")
    
28.     for each in targets:
    
29.         ranks.append(each.text)
    
30. 
    
31.     # 资料
    
32.     messages = []
    
33.     targets = soup.find_all("div", class_="bd")
    
34.     for each in targets:
    
35.         try:
    
36.             messages.append(each.p.text.split('\n')[1].strip() + each.p.text.split('\n')[2].strip())
    
37.         except:
    
38.             continue
    
39. 
    
40.     result = []
    
41.     length = len(movies)
    
42.     for i in range(length):
    
43.         result.append([movies[i] ,ranks[i] ,messages[i]])
    
44. 
    
45.     return result
    
46. 
    
47. 
    
48. # 找出一共有多少个页面
    
49. def find_depth(res):
    
50.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
51.     depth = soup.find('span', class_='next').previous_sibling.previous_sibling.text
    
52. 
    
53.     return int(depth)
    
54. 
    
55. def save_to_excel(result):
    
56.     wb = openpyxl.Workbook()
    
57.     ws = wb.active
    
58. 
    
59.     ws['A1'] = '电影名称'
    
60.     ws['B1'] = '评分'
    
61.     ws['C1'] = '资料'
    
62. 
    
63.     for each in result:
    
64.         ws.append(each)
    
65. 
    
66.     wb.save('豆瓣TOP250.xlsx')
    
67. 
    
68. 
    
69. def main():
    
70.     host = "https://movie.douban.com/top250"
    
71.     res = open_url(host)
    
72.     depth = find_depth(res)
    
73. 
    
74.     result = []
    
75.     for i in range(depth):
    
76.         url = host + '/?start=' + str(25 * i)
    
77.         res = open_url(url)
    
78.         result.extend(find_movies(res))
    
79.     save_to_excel(result)
    
80. 
    
81. 
    
82. if __name__ == "__main__":
    
83.     main()

复制代码