关于线程池的异常问题

风尘岁月

1. import requests
   
2. import parsel
   
3. 
   
4. headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3756.400 QQBrowser/10.5.4043.400'}
   
5. 
   
6. 
   
7. def send_requests(url):
   
8.     '''请求数据'''
   
9.     response = requests.get(url = url, headers = headers, verify = False)
   
10.     return(response)
    
11. 
    
12. def parse_data(data):
    
13.     '''数据解析'''
    
14.     selector = parsel.Selector(data)
    
15.     result_list = selector.xpath('//a[@class="col-xs-6 col-sm-3"]')
    
16.     for result in result_list:
    
17.         img_url = result.xpath('./img/@data-original').extract_first()
    
18.         img_title = result.xpath('./img/@alt').extract_first()
    
19. 
    
20.         #准备文件后缀名
    
21.         all_title = title + '.' + src_url.split('.')[-1]
    
22.         yield all_title, src_url
    
23. 
    
24. 
    
25. def sava_data(file_name,data):
    
26.     '''数据保存'''
    
27.     with open('img\\' + file_name, mode = 'wb') as f:
    
28.         f.write(data)
    
29.         print('保存完成:', file_name)
    
30. 
    
31. 
    
32. 
    
33. def main(page):
    
34.     '''实现翻页的效果'''
    
35.     for page in range(1,page + 1):
    
36.         print('============正在爬取第{}页数据============'.format(page))
    
37.         thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=3)
    
38.         res = send_request('https://www.doutula.com/photo/list/?page={}'.format(str(page)))
    
39.         src_url = parse_data(res.text)
    
40.         for file, url in src_url:
    
41.             image_response = send_request(url)
    
42.             hread_pool.submit(save_data,file,image_response.content)
    
43.         thread_pool.shutdown()
    
44. 
    
45. 
    
46. 
    
47. 
    
48. if __name__ == '__main__':
    
49.     main(10)

复制代码

报错内容：


C:\Users\Administrator\AppData\Local\Programs\Python\Python36\python.exe E:/python_fruit/表情包/表情包_ronot-多线程.py
Traceback (most recent call last):
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 49, in <module>
    main(10)
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 37, in main
    thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=3)
NameError: name 'concurrent' is not defined
============正在爬取第1页数据============

Process finished with exit code 1

'''最近比较闲 在学线程池 但是又层出不穷的BUG 希望各位鱼油能够给一些与线程有关的文献或者资料啥的'''

风尘岁月

我知道为什么我会报错了 我是少写了import concurrent.futures

风尘岁月

哪位鱼油能告诉我这个祖安异常是怎么回事？
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\python.exe E:/python_fruit/表情包/表情包_ronot-多线程.py
============正在爬取第1页数据============
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py:986: InsecureRequestWarning: Unverified HTTPS request is being made to host 'www.doutula.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/e ... e.html#ssl-warnings
  InsecureRequestWarning,
Traceback (most recent call last):
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 58, in <module>
    main(10)
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 48, in main
    image_response = send_request(url)
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 16, in send_request
    response = requests.get(url = url, headers = headers, verify = False)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 76, in get
    return request('get', url, params=params, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 516, in request
    prep = self.prepare_request(req)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 459, in prepare_request
    hooks=merge_hooks(request.hooks, self.hooks),
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 314, in prepare
    self.prepare_url(url, params)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 388, in prepare_url
    raise MissingSchema(error)
requests.exceptions.MissingSchema: Invalid URL '一群渣渣': No schema supplied. Perhaps you meant http://一群渣渣?

风尘岁月

目前遇到的祖安异常:
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\python.exe E:/python_fruit/表情包/表情包_ronot-多线程.py
============正在爬取第1页数据============
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py:986: InsecureRequestWarning: Unverified HTTPS request is being made to host 'www.doutula.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/e ... e.html#ssl-warnings
  InsecureRequestWarning,
Traceback (most recent call last):
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 58, in <module>
    main(10)
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 48, in main
    image_response = send_request(url)
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 16, in send_request
    response = requests.get(url = url, headers = headers, verify = False)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 76, in get
    return request('get', url, params=params, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 516, in request
    prep = self.prepare_request(req)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 459, in prepare_request
    hooks=merge_hooks(request.hooks, self.hooks),
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 314, in prepare
    self.prepare_url(url, params)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 388, in prepare_url
    raise MissingSchema(error)
requests.exceptions.MissingSchema: Invalid URL '一群渣渣': No schema supplied. Perhaps you meant http://一群渣渣?

Process finished with exit code 1

风尘岁月

目前写出来的最新的代码

1. import requests
   
2. import parsel
   
3. import re
   
4. import concurrent.futures
   
5. 
   
6. 
   
7. 
   
8. 
   
9. 
   
10. headers = {
    
11.     'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3756.400 QQBrowser/10.5.4043.400'}
    
12. 
    
13. 
    
14. def send_request(url):
    
15.     '''请求数据'''
    
16.     response = requests.get(url = url, headers = headers, verify = False)
    
17.     return(response)
    
18. 
    
19. def parse_data(data):
    
20.     '''数据解析'''
    
21.     selector = parsel.Selector(data)
    
22.     result_list = selector.xpath('//a[@class="col-xs-6 col-sm-3"]')
    
23.     for result in result_list:
    
24.         title = result.xpath('./img/@data-original').extract_first()
    
25.         src_url = result.xpath('./img/@alt').extract_first()
    
26. 
    
27.         #准备文件后缀名
    
28.         all_title = title + '.' + src_url.split('.')[-1]
    
29.         yield all_title, src_url
    
30. 
    
31. 
    
32. def sava_data(file_name,data):
    
33.     '''数据保存'''
    
34.     with open('img\\' + file_name, mode = 'wb') as f:
    
35.         f.write(data)
    
36.         print('保存完成:', file_name)
    
37. 
    
38. 
    
39. 
    
40. def main(page):
    
41.     '''实现翻页的效果'''
    
42.     for page in range(1,page + 1):
    
43.         print('============正在爬取第{}页数据============'.format(page))
    
44.         thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=3)
    
45.         res = send_request('https://www.doutula.com/photo/list/?page={}'.format(str(page)))
    
46.         src_url = parse_data(res.text)
    
47.         for file, url in src_url:
    
48.             image_response = send_request(url)
    
49.             thread_pool.submit(save_data, file, image_response.content)
    
50. 
    
51. 
    
52.         thread_pool.shutdown()
    
53. 
    
54. 
    
55. 
    
56. 
    
57. if __name__ == '__main__':
    
58.     main(10)

复制代码

各位鱼油可以自己尝试哈
有个异常：祖安异常
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\python.exe E:/python_fruit/表情包/表情包_ronot-多线程.py
============正在爬取第1页数据============
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py:986: InsecureRequestWarning: Unverified HTTPS request is being made to host 'www.doutula.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/e ... e.html#ssl-warnings
  InsecureRequestWarning,
Traceback (most recent call last):
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 58, in <module>
    main(10)
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 48, in main
    image_response = send_request(url)
  File "E:/python_fruit/表情包/表情包_ronot-多线程.py", line 16, in send_request
    response = requests.get(url = url, headers = headers, verify = False)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 76, in get
    return request('get', url, params=params, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 516, in request
    prep = self.prepare_request(req)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 459, in prepare_request
    hooks=merge_hooks(request.hooks, self.hooks),
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 314, in prepare
    self.prepare_url(url, params)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 388, in prepare_url
    raise MissingSchema(error)
requests.exceptions.MissingSchema: Invalid URL '一群渣渣': No schema supplied. Perhaps you meant http://一群渣渣?

Process finished with exit code 1


希望各位鱼油能够告诉我为什么会出现这个异常