PYTHON爬虫豆瓣250名代码出错

首虎

Traceback (most recent call last):
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\book lean\爬虫自我修养4 爬取豆瓣网评分.py", line 87, in <module>
    main()
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\book lean\爬虫自我修养4 爬取豆瓣网评分.py", line 69, in main
    res = open_url(host)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\book lean\爬虫自我修养4 爬取豆瓣网评分.py", line 11, in open_url
    res = requests.get(url, headers=headers, proxies=proxies)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\api.py", line 60, in request
    return session.request(method=method, url=url, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\sessions.py", line 533, in request
    resp = self.send(prep, **send_kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\sessions.py", line 646, in send
    r = adapter.send(request, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 412, in send
    conn = self.get_connection(request.url, proxies)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 309, in get_connection
    proxy_manager = self.proxy_manager_for(proxy)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 193, in proxy_manager_for
    manager = self.proxy_manager[proxy] = proxy_from_url(
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\poolmanager.py", line 470, in proxy_from_url
    return ProxyManager(proxy_url=url, **kw)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\poolmanager.py", line 420, in __init__
    raise ProxySchemeUnknown(proxy.scheme)
urllib3.exceptions.ProxySchemeUnknown: Not supported proxy scheme None


运行过程中出现这些问题  请小甲鱼帮助一下

zltzlt

把你的代码发上来！

zzzyf

下载看看

首虎

zltzlt 发表于 2020-2-12 17:22
把你的代码发上来！

1. import requests
   
2. import bs4
   
3. import re
   
4. import openpyxl
   
5. 
   
6. def open_url(url):
   
7.     # 使用代理
   
8.     proxies = {"http": "127.0.0.1:1080", "https": "127.0.0.1:1080"}
   
9.     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
   
10. 
    
11.     res = requests.get(url, headers=headers, proxies=proxies)
    
12.     res = requests.get(url, headers=headers)
    
13. 
    
14.     return res
    
15. 
    
16. def find_movies(res):
    
17.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
18. 
    
19.     # 电影名
    
20.     movies = []
    
21.     targets = soup.find_all("div", class_="hd")
    
22.     for each in targets:
    
23.         movies.append(each.a.span.text)
    
24. 
    
25.     # 评分
    
26.     ranks = []
    
27.     targets = soup.find_all("span", class_="rating_num")
    
28.     for each in targets:
    
29.         ranks.append(each.text)
    
30. 
    
31.     # 资料
    
32.     messages = []
    
33.     targets = soup.find_all("div", class_="bd")
    
34.     for each in targets:
    
35.         try:
    
36.             messages.append(each.p.text.split('\n')[1].strip() + each.p.text.split('\n')[2].strip())
    
37.         except:
    
38.             continue
    
39. 
    
40.     result = []
    
41.     length = len(movies)
    
42.     for i in range(length):
    
43.         result.append([movies[i], ranks[i], messages[i]])
    
44. 
    
45.     return result
    
46. 
    
47. # 找出一共有多少个页面
    
48. def find_depth(res):
    
49.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
50.     depth = soup.find('span', class_='next').previous_sibling.previous_sibling.text
    
51. 
    
52.     return int(depth)
    
53. 
    
54. def save_to_excel(result):
    
55.     wb = openpyxl.Workbook()
    
56.     ws = wb.active
    
57. 
    
58.     ws['A1'] = "电影名称"
    
59.     ws['B1'] = "评分"
    
60.     ws['C1'] = "资料"
    
61. 
    
62.     for each in result:
    
63.         ws.append(each)
    
64. 
    
65.     wb.save("豆瓣TOP250电影.xlsx")
    
66. 
    
67. def main():
    
68.     host = "https://movie.douban.com/top250"
    
69.     res = open_url(host)
    
70.     depth = find_depth(res)
    
71. 
    
72.     result = []
    
73.     for i in range(depth):
    
74.         url = host + '/?start=' + str(25 * i)
    
75.         res = open_url(url)
    
76.         result.extend(find_movies(res))
    
77. 
    
78.     '''
    
79.     with open("test.txt", "w", encoding="utf-8") as f:
    
80.         for each in result:
    
81.             f.write(each)
    
82.     '''
    
83. 
    
84.     save_to_excel(result)
    
85.    
    
86. if __name__ == "__main__":
    
87.     main()

复制代码

xiangzhihengkan

首虎 发表于 2020-2-13 10:27

1.你使用的代理是有问题的
2.你写了两行res = requests.get(url, headers=headers, proxies=proxies)
                  res = requests.get(url, headers=headers)

当去掉了第一行的res = ...... 代码正常运行

首虎

xiangzhihengkan 发表于 2020-2-13 22:31
1.你使用的代理是有问题的
2.你写了两行res = requests.get(url, headers=headers, proxies=proxies)
  ...

去掉第一行的res 一样报错

xiangzhihengkan

首虎 发表于 2020-2-13 23:13
去掉第一行的res 一样报错

复制你的代码，去掉以后是可以正常运行的

首虎

xiangzhihengkan 发表于 2020-2-14 09:04
复制你的代码，去掉以后是可以正常运行的

你是哪个版本的python

xiangzhihengkan

首虎 发表于 2020-2-14 15:37
你是哪个版本的python

3.7