用scrapy框架爬取“斗图啦”网站表情包

风尘岁月

要求：用scrapy框架爬取“斗图啦”的表情包
2.使用ImagePipline管道

异步的源码就在这:

1. import asyncio,aiohttp,os
   
2. from parsel import Selector
   
3. from time import time
   
4. 
   
5. 
   
6. headers = {
   
7.     'User-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'
   
8. }
   
9. 
   
10. 
    
11. path = os.path.dirname(__file__)
    
12. path = path + '/image'
    
13. if not os.path.exists(path):
    
14.     print('检测到没有容器')
    
15.     os.mkdir(path)
    
16.     print('生成完毕' + path)
    
17. 
    
18. 
    
19. class Download(object):
    
20. 
    
21.     def mk_url(self,startnum,endnum):
    
22.         for num in range(startnum,endnum+1):
    
23.             base_url = 'https://www.doutula.com/photo/list/?page={}'.format(num)
    
24.             task_list = []
    
25.             task_list.append(base_url)
    
26.             yield task_list
    
27. 
    
28. 
    
29. 
    
30. 
    
31.     async def fetch_html(self,session,url):
    
32.         '''请求网页数据'''
    
33.         async with session.get(url) as response:
    
34.             return await response.text()
    
35. 
    
36.     async def fetch_img(self,session,url):
    
37.         '''请求图片数据'''
    
38.         async with session.get(url) as data:
    
39.             return await data.read()
    
40. 
    
41.     async def parse_data(self,session,html):
    
42.         '''处理数据'''
    
43.         selector = Selector(html)
    
44.         result_list = selector.xpath('//a[@class="col-xs-6 col-sm-3"]')
    
45. 
    
46.         for result in result_list:
    
47.             img_url = result.xpath('./img/@data-original').extract_first()
    
48.             img_title = result.xpath('./img/@alt').extract_first()
    
49. 
    
50.             all_title = img_title + '.' + img_url.split('.')[-1]
    
51. 
    
52.             content = await self.fetch_img(session,img_url)
    
53. 
    
54.             try:
    
55.                 with open(path + "\" + all_title,mode='wb') as f:
    
56.                     print("下载完成:",all_title)
    
57.                     f.write(content)
    
58. 
    
59.             except Exception as e:
    
60.                 print(e)
    
61. 
    
62. 
    
63.     async def start_save(self,url):
    
64.         async with aiohttp.ClientSession(headers=headers) as session:
    
65.             html = await self.fetch_html(session,url)
    
66.             await self.parse_data(session = session, html = html)
    
67. 
    
68.     async def download_pictures(self,startnum,endnum):
    
69.         for page in range(startnum,endnum+1):
    
70.             print("######正在下载第{}页数据######".format(page))
    
71.             url_list = self.mk_url(startnum,endnum)
    
72.             for url in url_list:
    
73.                 base_url = url[0]
    
74.                 await self.start_save(base_url)
    
75. 
    
76. 
    
77. '''实例化'''
    
78. if __name__ == '__main__':
    
79.     print("任务启动中...")
    
80.     download = Download()
    
81.     loop = asyncio.get_event_loop()
    
82. 
    
83.     tasks = [
    
84.         asyncio.ensure_future(download.download_pictures(1,2000)),
    
85.         asyncio.ensure_future(download.download_pictures(2001, 4000)),
    
86.         
    
87.     ]
    
88.     start_time = time()
    
89.     loop.run_until_complete(asyncio.gather(*tasks))
    
90.     end_time = time()
    
91.     run_time = end_time - start_time
    
92.     print(run_time)

复制代码