scrapy持久化存储输出结果间隔一行None

wcq15759797758

1. import scrapy
   
2. from tutorial.items import QuoteItem
   
3. 
   
4. class QuotesSpider(scrapy.Spider):
   
5.     name = 'quotes'
   
6.     #allowed_domains = ['quoten.toscrape.com']
   
7.     start_urls = ['http://quotes.toscrape.com/']
   
8. 
   
9.     def parse(self, response):
   
10.         tr_list = response.xpath('//div[@class="col-md-8"]//div')
    
11.         for tr in tr_list:
    
12.             item = QuoteItem()
    
13.             item['text'] = tr.xpath('./span[@class="text"]/text()').extract_first()
    
14.             item['author'] = tr.xpath('./span/small[@class="author"]/text()').extract_first()
    
15.             item['tags'] = tr.xpath('./div/a/text()').extract()
    
16.             yield item

复制代码

wcq15759797758

这是Item的设置

1. import scrapy
   
2. 
   
3. class QuoteItem(scrapy.Item):
   
4.     #定义三个字段
   
5.     text = scrapy.Field()
   
6.     author = scrapy.Field()
   
7.     tags = scrapy.Field()
   

复制代码

wcq15759797758

这是pipelines

1. class TutorialPipeline:
   
2.     def process_item(self, item, spider):
   
3.         print(item)
   
4.         return item
   

复制代码

wcq15759797758

输出结果
{'author': 'Albert Einstein',
'tags': ['change', 'deep-thoughts', 'thinking', 'world'],
'text': '“The world as we have created it is a process of our thinking. It '
         'cannot be changed without changing our thinking.”'}
{'author': None, 'tags': [], 'text': None}
{'author': 'J.K. Rowling',
'tags': ['abilities', 'choices'],
'text': '“It is our choices, Harry, that show what we truly are, far more '
         'than our abilities.”'}
{'author': None, 'tags': [], 'text': None}
{'author': 'Albert Einstein',
'tags': ['inspirational', 'life', 'live', 'miracle', 'miracles'],
'text': '“There are only two ways to live your life. One is as though nothing '
         'is a miracle. The other is as though everything is a miracle.”'}
{'author': None, 'tags': [], 'text': None}
{'author': 'Jane Austen',
'tags': ['aliteracy', 'books', 'classic', 'humor'],
'text': '“The person, be it gentleman or lady, who has not pleasure in a good '
         'novel, must be intolerably stupid.”'}
{'author': None, 'tags': [], 'text': None}
{'author': 'Marilyn Monroe',
'tags': ['be-yourself', 'inspirational'],
'text': "“Imperfection is beauty, madness is genius and it's better to be "
         'absolutely ridiculous than absolutely boring.”'}
{'author': None, 'tags': [], 'text': None}
{'author': 'Albert Einstein',
'tags': ['adulthood', 'success', 'value'],
'text': '“Try not to become a man of success. Rather become a man of value.”'}
{'author': None, 'tags': [], 'text': None}
{'author': 'André Gide',
'tags': ['life', 'love'],
'text': '“It is better to be hated for what you are than to be loved for what '
         'you are not.”'}
{'author': None, 'tags': [], 'text': None}
{'author': 'Thomas A. Edison',
'tags': ['edison', 'failure', 'inspirational', 'paraphrased'],
'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
{'author': None, 'tags': [], 'text': None}
{'author': 'Eleanor Roosevelt',
'tags': ['misattributed-eleanor-roosevelt'],
'text': '“A woman is like a tea bag; you never know how strong it is until '
         "it's in hot water.”"}
{'author': None, 'tags': [], 'text': None}
{'author': 'Steve Martin',
'tags': ['humor', 'obvious', 'simile'],
'text': '“A day without sunshine is like, you know, night.”'}
{'author': None, 'tags': [], 'text': None}

wcq15759797758

wcq15759797758 发表于 2021-6-5 13:50
输出结果
{'author': 'Albert Einstein',
'tags': ['change', 'deep-thoughts', 'thinking', 'world'],
...

数据是爬取到了但是 爬取到数据之后下一行的数据全是空