关于scrapy中的CrawlSpider，如何添加headers？

syxuming

网上看到CrawSpider的例子如下

1. import scrapy
   
2. from scrapy.spiders import CrawlSpider, Rule
   
3. from scrapy.linkextractors import LinkExtractor
   
4. 
   
5. class MySpider(CrawlSpider):
   
6.     name = 'example.com'
   
7.     allowed_domains = ['example.com']
   
8.     start_urls = ['http://www.example.com']
   
9. 
   
10.     rules = (
    
11.         Rule(LinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
    
12.         Rule(LinkExtractor(allow=('item\.php', )), callback='parse_item'),
    
13.     )
    
14. 
    
15.     def parse_item(self, response):
    
16.         self.logger.info('Hi, this is an item page! %s', response.url)
    
17.         item = scrapy.Item()
    
18.         item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
    
19.         item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
    
20.         return item

复制代码

但是有些网站需要添加headers才能正常访问，那么CrawlSpider中如何去添加header呢

ba21

参考：

1. #!/usr/bin/env python
   
2. # -*- coding:utf-8 -*-
   
3. from scrapy.contrib.spiders import CrawlSpider, Rule
   
4. from scrapy.selector import Selector
   
5. from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
   
6. from scrapy.http import Request, FormRequest
   
7. from zhihu.items import ZhihuItem
   
8. 
   
9. 
   
10. 
    
11. class ZhihuSipder(CrawlSpider) :
    
12.     name = "zhihu"
    
13.     allowed_domains = ["www.zhihu.com"]
    
14.     start_urls = [
    
15.         "http://www.zhihu.com"
    
16.     ]
    
17.     rules = (
    
18.         Rule(SgmlLinkExtractor(allow = ('/question/\d+#.*?', )), callback = 'parse_page', follow = True),
    
19.         Rule(SgmlLinkExtractor(allow = ('/question/\d+', )), callback = 'parse_page', follow = True),
    
20.     )
    
21.     headers = {
    
22.     "Accept": "*/*",
    
23.     "Accept-Encoding": "gzip,deflate",
    
24.     "Accept-Language": "en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4",
    
25.     "Connection": "keep-alive",
    
26.     "Content-Type":" application/x-www-form-urlencoded; charset=UTF-8",
    
27.     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36",
    
28.     "Referer": "http://www.zhihu.com/"
    
29.     }
    
30. 
    
31.     #重写了爬虫类的方法, 实现了自定义请求, 运行成功后会调用callback回调函数
    
32.     def start_requests(self):
    
33.         return [Request("https://www.zhihu.com/login", meta = {'cookiejar' : 1}, callback = self.post_login)]
    
34. 
    
35.     #FormRequeset出问题了
    
36.     def post_login(self, response):
    
37.         print 'Preparing login'
    
38.         #下面这句话用于抓取请求网页后返回网页中的_xsrf字段的文字, 用于成功提交表单
    
39.         xsrf = Selector(response).xpath('//input[@name="_xsrf"]/@value').extract()[0]
    
40.         print xsrf
    
41.         #FormRequeset.from_response是Scrapy提供的一个函数, 用于post表单
    
42.         #登陆成功后, 会调用after_login回调函数
    
43.         return [FormRequest.from_response(response,   #"http://www.zhihu.com/login",
    
44.                             meta = {'cookiejar' : response.meta['cookiejar']},
    
45.                             headers = self.headers,  #注意此处的headers
    
46.                             formdata = {
    
47.                             '_xsrf': xsrf,
    
48.                             'email': '1095511864@qq.com',
    
49.                             'password': '123456'
    
50.                             },
    
51.                             callback = self.after_login,
    
52.                             dont_filter = True
    
53.                             )]
    
54. 
    
55.     def after_login(self, response) :
    
56.         for url in self.start_urls :
    
57.             yield self.make_requests_from_url(url)
    
58. 
    
59.     def parse_page(self, response):
    
60.         problem = Selector(response)
    
61.         item = ZhihuItem()
    
62.         item['url'] = response.url
    
63.         item['name'] = problem.xpath('//span[@class="name"]/text()').extract()
    
64.         print item['name']
    
65.         item['title'] = problem.xpath('//h2[@class="zm-item-title zm-editable-content"]/text()').extract()
    
66.         item['description'] = problem.xpath('//div[@class="zm-editable-content"]/text()').extract()
    
67.         item['answer']= problem.xpath('//div[@class=" zm-editable-content clearfix"]/text()').extract()
    
68.         return item
    
69. 
    

复制代码