[已解决]关于爬虫的一些看不懂后台文件

风尘岁月

放暑假了 最近闲的蛋疼 写了个爬虫结果一堆bug 想改成多线程加速吧 bug更多了
单线程版:

1. #导包
   
2. import requests
   
3. import parsel
   
4. import os
   
5. import time
   
6. 
   
7. 
   
8. #准备工作
   
9. if not os.path.exists('image'):
   
10.     os.mkdir('image')
    
11. 
    
12. headers = {
    
13.     'User-Agent':
    
14.         'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
    
15. }
    
16. 
    
17. #爬虫
    
18. for page in range(0,6500):
    
19.     #url,headers准备
    
20.     print("=================/正在保存第{}页数据=================".format(page))
    
21.     base_url = 'https://anime-pictures.net/pictures/view_posts/{}?lang=en'.format(page)
    
22. 
    
23.     if page>0:
    
24.         time.sleep(10)
    
25.         print('太累了，休息下|･ω･` )')
    
26. 
    
27.     #请求数据
    
28.     response = requests.get(url = base_url,headers = headers)
    
29.     html_data = response.text
    
30. 
    
31. 
    
32.     #筛选数据
    
33.     selector = parsel.Selector(html_data)
    
34.     result_list = selector.xpath('//span[@class="img_block_big"]')
    
35. 
    
36.     for result in result_list:
    
37.         image_url = result.xpath('./a/picture/source/img/@src').extract_first()
    
38.         image_id = result.xpath('./a/picture/source/img/@id').extract_first()
    
39. 
    
40.         img_url = 'https:' + image_url  # 手动拼接完整url
    
41. 
    
42.         all_title = image_id + '.' + img_url.split('.')[-1]
    
43. 
    
44.         img_data = requests.get(url = img_url,headers = headers).content
    
45. 
    
46. 
    
47.         #保存数据
    
48.         try:
    
49.             with open('image\\' + all_title, mode='wb') as f:
    
50.                 print('保存成功:', image_id)
    
51.                 f.write(img_data)
    
52. 
    
53.         except:
    
54.             pass
    
55.             print('保存失败:', image_id,'(•́へ•́╬)')
    
56.            
    
57. 
    
58. 
    
59. 
    

复制代码

异常内容:
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\python.exe E:/python_fruit/滑稽图/爬虫.py
=================/正在保存第0页数据=================
保存成功: common_preview_img_655082
保存成功: common_preview_img_655084
保存成功: common_preview_img_655085
保存成功: common_preview_img_654687
保存成功: common_preview_img_655061
保存成功: common_preview_img_655062
保存成功: common_preview_img_655063
保存成功: common_preview_img_655064
保存成功: common_preview_img_654930
保存成功: common_preview_img_654929
保存成功: common_preview_img_654927
保存成功: common_preview_img_654850
保存成功: common_preview_img_654926
Traceback (most recent call last):
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\contrib\pyopenssl.py", line 488, in wrap_socket
    cnx.do_handshake()
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\OpenSSL\SSL.py", line 1934, in do_handshake
    self._raise_ssl_error(self._ssl, result)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\OpenSSL\SSL.py", line 1663, in _raise_ssl_error
    raise SysCallError(errno, errorcode.get(errno))
OpenSSL.SSL.SysCallError: (10054, 'WSAECONNRESET')

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py", line 677, in urlopen
    chunked=chunked,
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py", line 381, in _make_request
    self._validate_conn(conn)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py", line 976, in _validate_conn
    conn.connect()
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connection.py", line 370, in connect
    ssl_context=context,
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\util\ssl_.py", line 377, in ssl_wrap_socket
    return context.wrap_socket(sock, server_hostname=server_hostname)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\contrib\pyopenssl.py", line 494, in wrap_socket
    raise ssl.SSLError("bad handshake: %r" % e)
ssl.SSLError: ("bad handshake: SysCallError(10054, 'WSAECONNRESET')",)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\adapters.py", line 449, in send
    timeout=timeout
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py", line 725, in urlopen
    method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\util\retry.py", line 439, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='cdn.anime-pictures.net', port=443): Max retries exceeded with url: /jvwall_images/956/9566c4f4db6d8c928273bc192fae2a40_cp.jpg (Caused by SSLError(SSLError("bad handshake: SysCallError(10054, 'WSAECONNRESET')",),))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "E:/python_fruit/滑稽图/爬虫.py", line 44, in <module>
    img_data = requests.get(url = img_url,headers = headers).content
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 76, in get
    return request('get', url, params=params, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 530, in request
    resp = self.send(prep, **send_kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\adapters.py", line 514, in send
    raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='cdn.anime-pictures.net', port=443): Max retries exceeded with url: /jvwall_images/956/9566c4f4db6d8c928273bc192fae2a40_cp.jpg (Caused by SSLError(SSLError("bad handshake: SysCallError(10054, 'WSAECONNRESET')",),))

Process finished with exit code 1



多线程版（多线程还没写好，先测试函数版单线程）

1. #导包
   
2. import requests
   
3. import time
   
4. import os
   
5. import threading
   
6. import parsel
   
7. 
   
8. 
   
9. if not os.path.exists('image'):
   
10.     os.mkdir('image')
    
11. 
    
12. 
    
13. base_url = 'https://anime-pictures.net/pictures/view_posts/0?lang=en'
    
14. 
    
15. headers = {
    
16.     'User-Agent':
    
17.         'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
    
18. }
    
19. 
    
20. cookie = {
    
21.     'GA1.2.1560346118.1594984735'
    
22. }
    
23. 
    
24. 
    
25. 
    
26. 
    
27. 
    
28. def get(url,headers):
    
29.     '''请求数据'''
    
30.     response = requests.get(url,headers)
    
31.     html_data = response.text
    
32.     return html_data
    
33. 
    
34. 
    
35. def parsel_data(html_data):
    
36.     '''筛选数据'''
    
37.     selector = parsel.Selector(html_data)
    
38.     result_list = selector.xpath('//span[@class="img_block_big"]')
    
39. 
    
40.     for result in result_list:
    
41.         image_url = result.xpath('./a/picture/source/img/@src').extract_first()
    
42.         image_id = result.xpath('./a/picture/source/img/@id').extract_first()
    
43. 
    
44.         img_url = 'https:' + image_url #手动拼url
    
45. 
    
46.         all_title = img_url
    
47. 
    
48.         img_data = requests.get(url = all_title,headers = headers).content
    
49. 
    
50. 
    
51.         yield all_title,image_id,img_data
    
52. 
    
53. 
    
54. def save(all_title,image_id,img_data):
    
55.     '''保存数据'''
    
56. 
    
57.     with open('image\\' + all_title, mode='wb') as f:
    
58.         print('保存成功:', image_id)
    
59.         #f.write(img_data)
    
60. 
    
61.     #except:
    
62.         #pass
    
63.         #print('保存失败:', img_id,'(&#8226;&#769;へ&#8226;&#769;╬)')
    
64. 
    
65. 
    
66. def sleep(time):
    
67.     '''休眠'''
    
68.     time.sleep(time)
    
69. 
    
70. 
    
71. 
    
72. 
    
73. if __name__ == '__main__':
    
74.     html_data = get(url=base_url, headers=headers)
    
75.     for image_data in parsel_data(html_data):
    
76.         all_title = image_data[0]  # url https://xxxxxxx...
    
77.         img_id = image_data[1]  # ID号
    
78.         img_data = image_data[2]  # 数据
    
79. 
    
80.         #print(all_title,img_id,img_data)
    
81. 
    
82.         save(all_title = all_title, image_id = img_id, img_data = img_data)
    
83. 
    
84. 异常内容:
    
85. （异常处理被我注释掉了 好弄出来报错内容 如果把注释去掉 则会一直显示保存失败）
    
86. C:\Users\Administrator\AppData\Local\Programs\Python\Python36\python.exe E:/python_fruit/滑稽图/爬虫_多线程.py
    
87. Traceback (most recent call last):
    
88.   File "E:/python_fruit/滑稽图/爬虫_多线程.py", line 82, in <module>
    
89.     save(all_title = all_title, image_id = img_id, img_data = img_data)
    
90.   File "E:/python_fruit/滑稽图/爬虫_多线程.py", line 57, in save
    
91.     with open('image\\' + all_title, mode='wb') as f:
    
92. OSError: [Errno 22] Invalid argument: 'image\\https://cdn.anime-pictures.net/jvwall_images/000/00064ebef030944d326648e00ba8aa07_cp.png'
    
93. 
    
94. Process finished with exit code 1
    
95. 
    
96. 
    
97. 还有 帮我修bug时 请注意带好纸巾 避免失血过多{:10_256:}
    
98. 最后 感谢各位鱼油的帮助{:10_334:}
    
99. 
    
100. 
     
101. 
     
102. 
     
103. 
     
104. 
     
105. 
     
106. 
     
107. 
     
108. 
     
109. 
     
110. 
     
111. 
     
112. 
     
113. 
     
114. 
     
115. 
     
116. 
     
117. 
     
118. 
     
119. 
     
120. 
     
121. 
     
122. 
     
123. 
     
124. 
     
125. 
     
126. 
     
127. 
     
128. 
     
129. 
     
130. 
     
131. 
     
132. 
     
133. 
     

复制代码

最佳答案

月排行榜 / 总排行榜

1q23w31

2020-8-7 07:06:09

风尘岁月 发表于 2020-7-21 20:32
ssl异常是没了 但是还有两个
：C:%users\Administrator\AppData\Local\Programs\Python\Python36\python ...

1. #导包
   
2. import requests
   
3. import parsel
   
4. import os
   
5. import time
   
6. 
   
7. 
   
8. #准备工作
   
9. if not os.path.exists('image'):
   
10.     os.mkdir('image')
    
11. 
    
12. headers = {
    
13.     'User-Agent':
    
14.         'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
    
15. }
    
16. 
    
17. #爬虫
    
18. for page in range(0,6500):
    
19.     #url,headers准备
    
20.     print("=================/正在保存第{}页数据=================".format(page))
    
21.     base_url = 'https://anime-pictures.net/pictures/view_posts/{}?lang=en'.format(page)
    
22. 
    
23.     if page>0:
    
24.         time.sleep(10)
    
25.         print('太累了，休息下|&#65381;ω&#65381;` )')
    
26. 
    
27.     #请求数据
    
28.     response = requests.get(url = base_url,headers = headers,verify = False)
    
29.     html_data = response.text
    
30. 
    
31. 
    
32.     #筛选数据
    
33.     selector = parsel.Selector(html_data)
    
34.     result_list = selector.xpath('//span[@class="img_block_big"]')
    
35. 
    
36.     for result in result_list:
    
37.         image_url = result.xpath('./a/picture/source/img/@src').extract_first()
    
38.         image_id = result.xpath('./a/picture/source/img/@id').extract_first()
    
39. 
    
40.         img_url = 'https:' + image_url  # 手动拼接完整url
    
41. 
    
42.         all_title = image_id + '.' + img_url.split('.')[-1]
    
43. 
    
44.         img_data = requests.get(url = img_url,headers = headers,verify = False).content
    
45. 
    
46. 
    
47.         #保存数据
    
48.         try:
    
49.             with open('image\\' + all_title, mode='wb') as f:
    
50.                 print('保存成功:', image_id)
    
51.                 f.write(img_data)
    
52. 
    
53.         except:
    
54.             pass
    
55.             print('保存失败:', image_id,'(&#8226;&#769;へ&#8226;&#769;╬)')
    
56.            
    

复制代码

单线程版，测试60张无报错，程序运行时有警告，忽略即可
如图

跳转到最佳答案楼层

Twilight6

试试在代码最前面加上 ：

1. import ssl
   
2. ssl._create_default_https_context = ssl._create_unverified_context

复制代码

风尘岁月

Twilight6 发表于 2020-7-21 19:18
试试在代码最前面加上 ：

ssl异常是没了 但是还有两个
：C:\Users\Administrator\AppData\Local\Programs\Python\Python36\python.exe E:/python_fruit/滑稽图/爬虫.py
=================/正在保存第0页数据=================
保存成功: common_preview_img_654943
保存成功: common_preview_img_655117
保存成功: common_preview_img_655118
保存成功: common_preview_img_654877
保存成功: common_preview_img_655119
保存成功: common_preview_img_654876
保存成功: common_preview_img_654875
Traceback (most recent call last):
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\contrib\pyopenssl.py", line 488, in wrap_socket
    cnx.do_handshake()
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\OpenSSL\SSL.py", line 1934, in do_handshake
    self._raise_ssl_error(self._ssl, result)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\OpenSSL\SSL.py", line 1663, in _raise_ssl_error
    raise SysCallError(errno, errorcode.get(errno))
OpenSSL.SSL.SysCallError: (10054, 'WSAECONNRESET')

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py", line 677, in urlopen
    chunked=chunked,
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py", line 381, in _make_request
    self._validate_conn(conn)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py", line 976, in _validate_conn
    conn.connect()
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connection.py", line 370, in connect
    ssl_context=context,
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\util\ssl_.py", line 377, in ssl_wrap_socket
    return context.wrap_socket(sock, server_hostname=server_hostname)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\contrib\pyopenssl.py", line 494, in wrap_socket
    raise ssl.SSLError("bad handshake: %r" % e)
ssl.SSLError: ("bad handshake: SysCallError(10054, 'WSAECONNRESET')",)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\adapters.py", line 449, in send
    timeout=timeout
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\connectionpool.py", line 725, in urlopen
    method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\urllib3\util\retry.py", line 439, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='cdn.anime-pictures.net', port=443): Max retries exceeded with url: /jvwall_images/25f/25f5a4e05c882ff0f6673432040a1328_cp.png (Caused by SSLError(SSLError("bad handshake: SysCallError(10054, 'WSAECONNRESET')",),))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "E:/python_fruit/滑稽图/爬虫.py", line 47, in <module>
    img_data = requests.get(url = img_url,headers = headers).content
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 76, in get
    return request('get', url, params=params, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 530, in request
    resp = self.send(prep, **send_kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 643, in send
    r = adapter.send(request, **kwargs)
  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\adapters.py", line 514, in send
    raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='cdn.anime-pictures.net', port=443): Max retries exceeded with url: /jvwall_images/25f/25f5a4e05c882ff0f6673432040a1328_cp.png (Caused by SSLError(SSLError("bad handshake: SysCallError(10054, 'WSAECONNRESET')",),))

Process finished with exit code 1

Twilight6

风尘岁月 发表于 2020-7-21 20:32
ssl异常是没了 但是还有两个
：C:%users\Administrator\AppData\Local\Programs\Python\Python36\python ...

其他的帮不了你了，没怎么遇到过~

风尘岁月

Twilight6 发表于 2020-7-21 20:33
其他的帮不了你了，没怎么遇到过~

好吧

1q23w31

风尘岁月 发表于 2020-7-21 20:32
ssl异常是没了 但是还有两个
：C:%users\Administrator\AppData\Local\Programs\Python\Python36\python ...

1. #导包
   
2. import requests
   
3. import parsel
   
4. import os
   
5. import time
   
6. 
   
7. 
   
8. #准备工作
   
9. if not os.path.exists('image'):
   
10.     os.mkdir('image')
    
11. 
    
12. headers = {
    
13.     'User-Agent':
    
14.         'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
    
15. }
    
16. 
    
17. #爬虫
    
18. for page in range(0,6500):
    
19.     #url,headers准备
    
20.     print("=================/正在保存第{}页数据=================".format(page))
    
21.     base_url = 'https://anime-pictures.net/pictures/view_posts/{}?lang=en'.format(page)
    
22. 
    
23.     if page>0:
    
24.         time.sleep(10)
    
25.         print('太累了，休息下|&#65381;ω&#65381;` )')
    
26. 
    
27.     #请求数据
    
28.     response = requests.get(url = base_url,headers = headers,verify = False)
    
29.     html_data = response.text
    
30. 
    
31. 
    
32.     #筛选数据
    
33.     selector = parsel.Selector(html_data)
    
34.     result_list = selector.xpath('//span[@class="img_block_big"]')
    
35. 
    
36.     for result in result_list:
    
37.         image_url = result.xpath('./a/picture/source/img/@src').extract_first()
    
38.         image_id = result.xpath('./a/picture/source/img/@id').extract_first()
    
39. 
    
40.         img_url = 'https:' + image_url  # 手动拼接完整url
    
41. 
    
42.         all_title = image_id + '.' + img_url.split('.')[-1]
    
43. 
    
44.         img_data = requests.get(url = img_url,headers = headers,verify = False).content
    
45. 
    
46. 
    
47.         #保存数据
    
48.         try:
    
49.             with open('image\\' + all_title, mode='wb') as f:
    
50.                 print('保存成功:', image_id)
    
51.                 f.write(img_data)
    
52. 
    
53.         except:
    
54.             pass
    
55.             print('保存失败:', image_id,'(&#8226;&#769;へ&#8226;&#769;╬)')
    
56.            
    

复制代码

单线程版，测试60张无报错，程序运行时有警告，忽略即可
如图