[已解决]scrapy爬虫yield重定向的问题

shentianbusi

1. # -*- coding: utf-8 -*-
   
2. import scrapy
   
3. from amazoncar import items
   
4. from amazoncar.items import AmazoncarItem
   
5. from scrapy.http import Request
   
6. import re
   
7. import time
   
8. 
   
9. 
   
10. 
    
11. class AmazonSpider(scrapy.Spider):
    
12.     name = 'amazon'
    
13.     allowed_domains = ['amazon.com']
    
14.     # start_urls = ['https://www.amazon.com/b/ref=sr_aj?node=10677469011&bbn=10677469011&ajr=0']    首先用这个爬取各类汽车分类界面地址
    
15. 
    
16.     def start_requests(self):
    
17.         urls=['https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_0?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710192011&ie=UTF8',
    
18.               'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_1?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710193011&ie=UTF8',
    
19.               'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_2?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710199011&ie=UTF8']
    
20.               # 'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_3?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710198011&ie=UTF8',
    
21.               # 'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_4?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710200011&ie=UTF8',
    
22.               # 'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_5?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710195011&ie=UTF8',
    
23.               # 'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_6?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710201011&ie=UTF8',
    
24.               # 'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_7?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710197011&ie=UTF8',
    
25.               # 'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_8?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710194011&ie=UTF8',
    
26.               # 'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_9?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710196011&ie=UTF8',
    
27.               # 'https://www.amazon.com/s/ref=lp_10677469011_nr_p_n_feature_four_bro_10?fst=as%3Aoff&rh=n%3A10677469011%2Cp_n_feature_four_browse-bin%3A11710202011&&ie=UTF8']
    
28. 
    
29.         pages=[26,71,20,20,16,9,14,21,140,85,19]
    
30.         #pages=[3,3,3,3,3,3,3,3,3,3,3]
    
31. 
    
32. 
    
33. 
    
34.         for url in urls:
    
35.             # for i in range(1,pages[i]+1):#11
    
36.             #     print(url)
    
37.             #     starturl=url+'&page={}'.format(i)
    
38.             #     if i==30 or i==60 or i==90 or i==120:
    
39.             #         time.sleep(10)
    
40.             #     else:
    
41.             #         pass
    
42.             #     yield scrapy.Request(url=starturl, callback=self.parse)
    
43.             yield  scrapy.Request(url,callback=self.parse,dont_filter=True)
    
44. 
    
45. 
    
46. 
    
47. 
    
48.     def parse(self, response):  #爬虫逻辑
    
49.         # print(response.text[0:100])
    
50.         # addlist = []
    
51.         # str1='https://www.amazon.com'
    
52.         # for i in range(1,12):
    
53.         #     addpath=response.xpath('//*[@id="leftNav"]/ul[1]/div/li[{}]/span/span'.format(i)).re("/s.*rnid=11710191011")
    
54.         #     addlist.append(str1+addpath[0])
    
55.         # print(addlist)   #验证是否成功取得地址
    
56.         amzitem = AmazoncarItem()
    
57.         maxpage=response.xpath('//*[@id="pagn"]/span[6]').re('>.*<') #.re("data='.*'"))
    
58. 
    
59.         if   maxpage:
    
60.             maxpage=maxpage[0][1:-1]
    
61.         else:
    
62.             print(response.url)
    
63.             return 0
    
64. 
    
65.         #页面产品代号
    
66.         m=response.xpath('//*[@id="s-result-count"]/text()').re('\d{1,3}')
    
67.         # print(type(m))
    
68.         # print(m)
    
69.         startnum=int(m[0])-1
    
70.         endnum=int(m[1])
    
71.         # print(startnum,endnum)
    
72. 
    
73.         for n in range(startnum,endnum):#24
    
74.             amzitem['body_Style']=response.xpath('//*[@id="s-result-count"]/span/span/text()').extract_first('无显示')
    
75.             amzitem['link']=response.xpath('//*[@id="result_{}"]/div/div[3]/div[1]/a/@href'.format(n)).extract_first('无显示')
    
76.             maker=response.xpath('//*[@id="result_{}"]/div/div[3]/div[2]/span[2]/text()'.format(n)).extract_first('无显示')
    
77.             amzitem['maker']=maker
    
78.             print(maker)
    
79.             #cats是读取的目标，首先用正则取得year
    
80.             cats=response.xpath('//*[@id="result_{}"]/div/div[3]/div[1]/a'.format(n)).re('title=".*" h')[0][7:-3]
    
81.             listmodel=cats.split(' ',1)
    
82.             amzitem['year']=listmodel[0]
    
83.             models=listmodel[1].split(maker+' ')[1]
    
84.             amzitem['model']=models
    
85.             amzitem['reviews']=response.xpath('//*[@id="result_{}"]/div/div[6]/a/text()'.format(n)).extract_first('无显示')
    
86.             if response.xpath('//*[@id="result_{}"]/div/div[6]/span/span/a/i[1]/span/text()'.format(n)).extract_first():
    
87.                 rating=response.xpath('//*[@id="result_{}"]/div/div[6]/span/span/a/i[1]/span/text()'.format(n)).extract_first()
    
88.                 rating=rating.split(' ')
    
89.                 amzitem['rating']=rating[0]
    
90.             else:
    
91.                 amzitem['rating'] = response.xpath('//*[@id="result_{}"]/div/div[6]/span/span/a/i[1]/span/text()'.format(n)).extract_first('无显示')
    
92. 
    
93.             yield amzitem
    
94.         next_url=response.xpath('//*[@id="pagnNextLink"]/@href').extract_first()
    
95.         if  next_url:
    
96.             yield Request('http://www.amazon.com'+next_url,callback=self.parse,dont_filter=True)

复制代码

这是主程序的代码，按编码应该是从初始请求网址列表中，先选择其中一个网址然后爬完该网址下每个下一页，然后返回爬下个列表中网址。
结果代码结果显示爬取的顺序是乱序的，这个爬会爬那个，求大神看下，问题出在哪里

最佳答案

月排行榜 / 总排行榜

wongyusing

2019-2-20 11:52:27

scrapy本来就是异步爬取的。
如果想要爬取的结果是有序的，那就给数据加个page字段。  
等数据全部爬取下来后，后期再进行索引排序

跳转到最佳答案楼层

兀颜光

yield scrapy.Request()

_谪仙

scrapy不是同步请求的，

wongyusing

scrapy本来就是异步爬取的。
如果想要爬取的结果是有序的，那就给数据加个page字段。  
等数据全部爬取下来后，后期再进行索引排序