如何制造多协程爬虫

风尘岁月

最近学习了gevent
想做多协程
可是不知道咋做（做了很多次 都各种异常(其实我不知道咋搞)）
代码如下:

1. from parsel import Selector
   
2. from random import randint
   
3. from time import time
   
4. import os,aiohttp,gevent
   
5. 
   
6. 
   
7. headers = {
   
8.     'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
   
9. 
   
10. }
    
11. 
    
12. if not os.path.exists("image"):
    
13.     os.mkdir("image")
    
14. 
    
15. class Download(object):
    
16. 
    
17.     def mk_url(self,startnum,endnum):
    
18.         '''生成url'''
    
19.         for _ in range(startnum,endnum+1):
    
20.             base_url = 'https://anime-pictures.net/pictures/view_posts/{}?lang=en'.format(_)
    
21.             task_list =[]
    
22.             task_list.append(base_url)
    
23.             return task_list
    
24. 
    
25.     async def fetch_html(self,session, url):
    
26.        '''请求网页数据'''
    
27.        async with session.get(url) as response:
    
28.            return await response.text()
    
29. 
    
30.     async def fetch_img_data(self,session,url):
    
31.         '''请求图片数据'''
    
32.         async with session.get(url) as data:
    
33.             return await data.read()
    
34. 
    
35.     async def parser_data(self,session,html):
    
36.         '''处理数据'''
    
37.         selector = Selector(html)
    
38.         result_list = selector.xpath('//span[@class="img_block_big"]')
    
39. 
    
40.         for result in result_list:
    
41.             image_url = result.xpath('./a/picture/source/img/@src').extract_first()
    
42.             img_url = 'https:' + image_url  # 手动拼url
    
43.             content = await self.fetch_img_data(session,img_url)
    
44. 
    
45.             id = str(randint(0,99999999999999))
    
46.             try:
    
47.                 with open('image\\' + id + os.path.splitext(img_url)[1], mode='wb') as f:
    
48.                     f.write(content)
    
49.                     print('保存完成',id)
    
50.             except Exception as e:
    
51.                 print(e)
    
52. 
    
53.     async def start_save(self,url):
    
54.         async with aiohttp.ClientSession(headers=headers) as session:
    
55.             html = await self.fetch_html(session, url=url)
    
56.             await self.parser_data(session=session, html=html)
    
57. 
    
58.     # base_url = 'https://anime-pictures.net/pictures/view_posts/0?lang=en'
    
59. 
    
60.     async def download_pictures(self,startnum,endnum):
    
61.         for _ in range(startnum,endnum+1):
    
62.             url_list = self.mk_url(startnum,endnum)
    
63.             for url in url_list:
    
64.                 base_url = url
    
65.                 await self.start_save(base_url)
    
66. 
    
67. '''实例化'''
    
68. if __name__ == '__main__':
    
69.     print('任务启动中...')
    
70.     download = Download()
    
71.     s_time = time()
    
72.     g1 = gevent.spawn(download.download_pictures,1,2000),
    
73.     g2 = gevent.spawn(download.download_pictures, 2001, 4000),
    
74.     # g3 = await gevent.spawn(download.download_pictures, 4001, 6000),
    
75.     # g4 = await gevent.spawn(download.download_pictures, 6001, 8000),
    
76.     gevent.joinall([g1,g2])
    
77. 
    
78.     e_time = time()
    
79.     print('用时:',e_time - s_time,'秒')
    
80. 
    
81. 
    

复制代码

改进出多协程就有重赏
谢谢各位鱼油

风尘岁月

1. from parsel import Selector
   
2. from random import randint
   
3. from time import time
   
4. import os,aiohttp,asyncio
   
5. 
   
6. 
   
7. headers = {
   
8.     'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
   
9. 
   
10. }
    
11. 
    
12. path = os.path.dirname(__file__)
    
13. path = path + '/image'
    
14. if not os.path.exists(path):
    
15.     print('检测到没有容器')
    
16.     os.mkdir(path)
    
17.     print("生成完毕" + path)
    
18. 
    
19. 
    
20. 
    
21. 
    
22. class Download(object):
    
23. 
    
24.     def mk_url(self,startnum,endnum):
    
25.         '''生成url'''
    
26.         for _ in range(startnum,endnum+1):
    
27.             base_url = 'https://anime-pictures.net/pictures/view_posts/{}?lang=en'.format(_)
    
28.             task_list =[]
    
29.             task_list.append(base_url)
    
30.             return task_list
    
31. 
    
32.     async def fetch_html(self,session, url):
    
33.        '''请求网页数据'''
    
34.        async with session.get(url) as response:
    
35.            return await response.text()
    
36. 
    
37.     async def fetch_img_data(self,session,url):
    
38.         '''请求图片数据'''
    
39.         async with session.get(url) as data:
    
40.             return await data.read()
    
41. 
    
42.     async def parser_data(self,session,html):
    
43.         '''处理数据'''
    
44.         selector = Selector(html)
    
45.         result_list = selector.xpath('//span[@class="img_block_big"]')
    
46. 
    
47.         for result in result_list:
    
48.             image_url = result.xpath('./a/picture/source/img/@src').extract_first()
    
49.             img_url = 'https:' + image_url  # 手动拼url
    
50.             content = await self.fetch_img_data(session,img_url)
    
51. 
    
52.             id = str(randint(0,99999999999999))
    
53.             try:
    
54.                 with open(path + '\\' + id + os.path.splitext(img_url)[1], mode='wb') as f:
    
55.                     f.write(content)
    
56.                     print('保存完成',id)
    
57.             except Exception as e:
    
58.                 print(e)
    
59. 
    
60.     async def start_save(self,url):
    
61.         async with aiohttp.ClientSession(headers=headers) as session:
    
62.             html = await self.fetch_html(session, url=url)
    
63.             await self.parser_data(session=session, html=html)
    
64. 
    
65.     # base_url = 'https://anime-pictures.net/pictures/view_posts/0?lang=en'
    
66. 
    
67.     async def download_pictures(self,startnum,endnum):
    
68.         for page in range(startnum,endnum+1):
    
69.             print("######正在下载第{}页数据######".format(page))
    
70.             url_list = self.mk_url(startnum,endnum)
    
71.             for url in url_list:
    
72.                 base_url = url
    
73.                 await self.start_save(base_url)
    
74. 
    
75. '''实例化'''
    
76. if __name__ == '__main__':
    
77.     print('任务启动中...')
    
78.     download = Download()
    
79.     loop = asyncio.get_event_loop()
    
80.     s_time = time()
    
81.     tasks = [
    
82.         asyncio.ensure_future(download.download_pictures(1,2000)),
    
83.         asyncio.ensure_future(download.download_pictures(2001,4000)),
    
84.         asyncio.ensure_future(download.download_pictures(4001, 6000))
    
85.     ]
    
86.     loop.run_until_complete(asyncio.gather(*tasks))
    
87.     e_time = time()
    
88.     print('用时:',e_time - s_time,'秒')
    
89. 
    
90. 
    

复制代码

风尘岁月

这么久过去了 难道就没有大佬能解答了吗
我都自己写好了

疾风怪盗

风尘岁月 发表于 2020-9-18 20:34
这么久过去了 难道就没有大佬能解答了吗
我都自己写好了

没听过协程，只知道进程、线程
如果你自己写好了，就分享一下代码呗

风尘岁月

疾风怪盗 发表于 2020-9-18 20:38
没听过协程，只知道进程、线程
如果你自己写好了，就分享一下代码呗

好吧
(╯▽╰)