scrapy

哈岁NB

tu.py

1. import scrapy
   
2. 
   
3. from ..items import DeepimgproItem
   
4. class ImgSpider(scrapy.Spider):
   
5.     name = 'img'
   
6.     # allowed_domains = ['www.xxx.com']
   
7.     start_urls = ['https://pic.netbian.com/4kmeinv/']
   
8. 
   
9.     def parse(self, response):
   
10.         #解析出了图片的名称和详情页的url
    
11.         li_list = response.xpath('//*[@id="main"]/div[3]/ul/li')
    
12.         for li in li_list:
    
13.             title = li.xpath('./a/b/text()').extract_first() + '.jpg'
    
14.             detail_url = 'https://pic.netbian.com'+li.xpath('./a/@href').extract_first()
    
15.             item = DeepimgproItem()
    
16.             item['title'] = title
    
17.             #需要对详情页的url发起请求，在详情页中获取图片的下载链接
    
18.             yield scrapy.Request(url=detail_url,callback=self.detail_parse,meta={'item':item})
    
19. 
    
20.     #解析详情页的数据
    
21.     def detail_parse(self,response):
    
22.         meta = response.meta
    
23.         item = meta['item']
    
24.         img_src = 'https://pic.netbian.com'+response.xpath('//*[@id="img"]/img/@src').extract_first()
    
25.         item['img_src'] = img_src
    
26. 
    
27.         yield item

复制代码

管道类

1. # Define your item pipelines here
   
2. #
   
3. # Don't forget to add your pipeline to the ITEM_PIPELINES setting
   
4. # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
   
5. 
   
6. 
   
7. # useful for handling different item types with a single interface
   
8. import scrapy
   
9. from itemadapter import ItemAdapter
   
10. 
    
11. from scrapy.pipelines.images import ImagesPipeline
    
12. class DeepimgproPipeline(ImagesPipeline):
    
13.     # def process_item(self, item, spider):
    
14.     #     return item
    
15.     def get_media_requests(self, item, info):
    
16.         img_src = item['img_src']
    
17.         #请求传参，将item中的图片名称传递给file_path方法
    
18.         #meta会将自身传递给file_path
    
19.         print(item['title'],'保存下载成功！')
    
20.         yield scrapy.Request(url=img_src,meta={'title':item['title']})
    
21.     def file_path(self, request, response=None, info=None, *, item=None):
    
22.         #返回图片的名称
    
23.         #接收请求传参过来的数据
    
24.         title = request.meta['title']
    
25.         return title
    
26.     def item_completed(self, results, item, info):
    
27.         return item
    

复制代码

大佬们，上面这个为什么爬到的是大图，下面却是缩略图
tu.py

1. import scrapy
   
2. from ..items import TupianItem
   
3. 
   
4. class TuSpider(scrapy.Spider):
   
5.     name = 'tu'
   
6.     #allowed_domains = ['www.xxx.com']
   
7.     start_urls = ['https://pic.netbian.com/4kmeinv/']
   
8. 
   
9.     def parse(self, response):
   
10.         li_list = response.xpath('//*[@id="main"]/div[3]/ul/li')
    
11.         for li in li_list:
    
12.             img_src = "https://pic.netbian.com" + li.xpath('.//img/@src').extract_first()
    
13. 
    
14.             item = TupianItem()
    
15.             item['src'] = img_src
    
16. 
    
17.             yield item

复制代码

   
管道类

1. # Define your item pipelines here
   
2. #
   
3. # Don't forget to add your pipeline to the ITEM_PIPELINES setting
   
4. # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
   
5. import scrapy
   
6. # useful for handling different item types with a single interface
   
7. from itemadapter import ItemAdapter
   
8. 
   
9. from scrapy.pipelines.images import ImagesPipeline
   
10. class mediaPileline(ImagesPipeline):
    
11. 
    
12.     #重写三个方法
    
13.     #获取图片数据
    
14.     def get_media_requests(self, item, info):
    
15.         img_src = item['src']
    
16.         print(img_src)
    
17.         yield scrapy.Request(img_src)
    
18.     #指定图片存储路径（只需返回图片名称）
    
19.     def file_path(self, request, response=None, info=None, *, item=None):
    
20.         imgName = request.url.split('/')[-1]
    
21.         print(imgName,'保存成功')
    
22.         return imgName
    
23.     def item_completed(self, results, item, info):
    
24.         return item
    

复制代码

哈岁NB

解决了