用scrapy爬豆瓣top250的问题

xiaosi4081

1. import scrapy
   
2. from douban.items import DoubanItem
   
3. 
   
4. class DoubanSpiderSpider(scrapy.Spider):
   
5.     name = 'douban'
   
6.     allowed_domains = ['douban.com']
   
7.     start_urls = ['https://movie.douban.com/top250']
   
8. 
   
9.     def parse(self, response):
   
10.         content = response.xpath('//ol/li')
    
11.         
    
12.         for each in content:
    
13.             item = DoubanItem()
    
14.             item['rank'] = each.xpath('//em/text()').extract()
    
15.             item['name'] = each.xpath('//div[@class="hd"]/a/span[1]/text()').extract()
    
16.             item['info'] = each.xpath('//p/text()').extract()
    
17.             item['scort'] = each.xpath('//div[@class="star"]/span[2]/text()').extract()
    
18. 
    
19.             yield item
    
20. 
    
21.         if len(response.xpath('//span[@class="next"]/a/@href')) != 0 :
    
22.             next_url = response.xpath('//span[@class="next"]/a/@href').extract()[0]
    
23. 
    
24.             yield scrapy.Request('https://movie.douban.com/top250' + next_url,callback = self.parse,dont_filter = False)

复制代码