[已解决]用scrapy框架爬花瓣

mgiftdc

学了scrapy后，参照网上的scrapy爬煎蛋写了个爬花瓣的，但一直爬不出来，找了好久原因都没找到，大神帮我看一下吧：
huaban_spider.py

1. import scrapy
   
2. from huaban.items import HuabanItem
   
3. from scrapy.crawler import CrawlerProcess
   
4. 
   
5. class HuabanSpider(scrapy.spiders.Spider):
   
6.     name = "huaban"
   
7.     allow_domains = ["huaban.com"]
   
8.     start_urls = ["http://huaban.com/pins/872365966"]
   
9.     def parse(self,response):
   
10.         item = HuabanItem()
    
11.         item['image_urls'] = response.xpath('//div[@class="main-image"]//div/a/img/@src').extract()
    
12.         print('image_urls',item['image_urls'])
    
13.         yield item
    
14.         new_url = response.xpath('//div[@id="board_pins_waterfall"]/a/@href').extract_first()
    
15.         if new_url:
    
16.             yield scrapy.Request(new_url,callback=self.parse)
    

复制代码

pipelines.py

1. # -*- coding: utf-8 -*-
   
2. 
   
3. # Define your item pipelines here
   
4. #
   
5. # Don't forget to add your pipeline to the ITEM_PIPELINES setting
   
6. # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
   
7. import os
   
8. import urllib
   
9. 
   
10. from huaban import settings
    
11. 
    
12. class HuabanPipeline(object):
    
13.     def process_item(self,item,spider):
    
14.         dir_path = '%s/%s'%(settings.IMAGES_STORE,spider.name)
    
15.         print("dir_path",dir_path)
    
16.         if not os.path.exists(dir_path):
    
17.             os.makedirs(dir_path)
    
18.         for image_url in item['image_urls']:
    
19.             list_name = image_url.split('/')
    
20.             file_name = list_name[len(list_name)-1]
    
21.             file_path = '%s/%s'%(dir_path,file_name)
    
22.             if os.path.exists(file_name):
    
23.                 continue
    
24.             with open(file_path,'wb') as file_writer:
    
25.                 conn = urllib.request.urlopen(image_url)
    
26.                 file_write.write(conn.read())
    
27.             file_writer.close()
    
28.         return item
    

复制代码

items.py

1. # -*- coding: utf-8 -*-
   
2. 
   
3. # Define here the models for your scraped items
   
4. #
   
5. # See documentation in:
   
6. # http://doc.scrapy.org/en/latest/topics/items.html
   
7. 
   
8. import scrapy
   
9. 
   
10. 
    
11. class HuabanItem(scrapy.Item):
    
12.     # define the fields for your item here like:
    
13.     # name = scrapy.Field()
    
14.     image_urls = scrapy.Field()
    

复制代码

settings.py

1. # -*- coding: utf-8 -*-
   
2. 
   
3. # Scrapy settings for huaban project
   
4. #
   
5. # For simplicity, this file contains only settings considered important or
   
6. # commonly used. You can find more settings consulting the documentation:
   
7. #
   
8. #     http://doc.scrapy.org/en/latest/topics/settings.html
   
9. #     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
   
10. #     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
    
11. 
    
12. BOT_NAME = 'huaban'
    
13. 
    
14. SPIDER_MODULES = ['huaban.spiders']
    
15. NEWSPIDER_MODULE = 'huaban.spiders'
    
16. 
    
17. 
    
18. # Crawl responsibly by identifying yourself (and your website) on the user-agent
    
19. #USER_AGENT = 'huaban (+http://www.yourdomain.com)'
    
20. 
    
21. # Obey robots.txt rules
    
22. ROBOTSTXT_OBEY = True
    
23. 
    
24. # Configure maximum concurrent requests performed by Scrapy (default: 16)
    
25. #CONCURRENT_REQUESTS = 32
    
26. 
    
27. # Configure a delay for requests for the same website (default: 0)
    
28. # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
    
29. # See also autothrottle settings and docs
    
30. #DOWNLOAD_DELAY = 3
    
31. # The download delay setting will honor only one of:
    
32. #CONCURRENT_REQUESTS_PER_DOMAIN = 16
    
33. #CONCURRENT_REQUESTS_PER_IP = 16
    
34. 
    
35. # Disable cookies (enabled by default)
    
36. #COOKIES_ENABLED = False
    
37. 
    
38. # Disable Telnet Console (enabled by default)
    
39. #TELNETCONSOLE_ENABLED = False
    
40. 
    
41. # Override the default request headers:
    
42. #DEFAULT_REQUEST_HEADERS = {
    
43. #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    
44. #   'Accept-Language': 'en',
    
45. #}
    
46. 
    
47. # Enable or disable spider middlewares
    
48. # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
    
49. #SPIDER_MIDDLEWARES = {
    
50. #    'huaban.middlewares.MyCustomSpiderMiddleware': 543,
    
51. #}
    
52. 
    
53. # Enable or disable downloader middlewares
    
54. # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
    
55. #DOWNLOADER_MIDDLEWARES = {
    
56. #    'huaban.middlewares.MyCustomDownloaderMiddleware': 543,
    
57. #}
    
58. 
    
59. # Enable or disable extensions
    
60. # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
    
61. #EXTENSIONS = {
    
62. #    'scrapy.extensions.telnet.TelnetConsole': None,
    
63. #}
    
64. 
    
65. # Configure item pipelines
    
66. # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
    
67. ITEM_PIPELINES = {
    
68.     'huaban.pipelines.HuabanPipeline': 1,
    
69. }
    
70. IMAGES_STORE = 'D:'
    
71. DOWNLOAD_DELAY = 0.25
    
72. # Enable and configure the AutoThrottle extension (disabled by default)
    
73. # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
    
74. #AUTOTHROTTLE_ENABLED = True
    
75. # The initial download delay
    
76. #AUTOTHROTTLE_START_DELAY = 5
    
77. # The maximum download delay to be set in case of high latencies
    
78. #AUTOTHROTTLE_MAX_DELAY = 60
    
79. # The average number of requests Scrapy should be sending in parallel to
    
80. # each remote server
    
81. #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
    
82. # Enable showing throttling stats for every response received:
    
83. #AUTOTHROTTLE_DEBUG = False
    
84. 
    
85. # Enable and configure HTTP caching (disabled by default)
    
86. # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
    
87. #HTTPCACHE_ENABLED = True
    
88. #HTTPCACHE_EXPIRATION_SECS = 0
    
89. #HTTPCACHE_DIR = 'httpcache'
    
90. #HTTPCACHE_IGNORE_HTTP_CODES = []
    
91. #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
    

复制代码

爬出来的结果是这样的
C:\Users\Administrator\huaban>scrapy crawl huaban
2016-09-30 22:24:52 [scrapy] INFO: Scrapy 1.1.3 started (bot: huaban)
2016-09-30 22:24:52 [scrapy] INFO: Overridden settings: {'DOWNLOAD_DELAY': 0.25,
'SPIDER_MODULES': ['huaban.spiders'], 'BOT_NAME': 'huaban', 'ROBOTSTXT_OBEY': T
rue, 'NEWSPIDER_MODULE': 'huaban.spiders'}
2016-09-30 22:24:52 [scrapy] INFO: Enabled extensions:
['scrapy.extensions.corestats.CoreStats',
'scrapy.extensions.logstats.LogStats',
'scrapy.extensions.telnet.TelnetConsole']
2016-09-30 22:24:53 [scrapy] INFO: Enabled downloader middlewares:
['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
'scrapy.downloadermiddlewares.retry.RetryMiddleware',
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware',
'scrapy.downloadermiddlewares.stats.DownloaderStats']
2016-09-30 22:24:53 [scrapy] INFO: Enabled spider middlewares:
['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
'scrapy.spidermiddlewares.referer.RefererMiddleware',
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
'scrapy.spidermiddlewares.depth.DepthMiddleware']
2016-09-30 22:24:53 [scrapy] INFO: Enabled item pipelines:
['huaban.pipelines.HuabanPipeline']
2016-09-30 22:24:53 [scrapy] INFO: Spider opened
2016-09-30 22:24:53 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 i
tems (at 0 items/min)
2016-09-30 22:24:53 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023
2016-09-30 22:24:53 [scrapy] DEBUG: Crawled (200) <GET http://huaban.com/robots.
txt> (referer: None)
2016-09-30 22:24:54 [scrapy] DEBUG: Crawled (200) <GET http://huaban.com/pins/87
2365966> (referer: None)
image_urls []
dir_path D:/huaban
2016-09-30 22:24:54 [scrapy] DEBUG: Scraped from <200 http://huaban.com/pins/872
365966>
{'image_urls': []}
2016-09-30 22:24:54 [scrapy] INFO: Closing spider (finished)
2016-09-30 22:24:54 [scrapy] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 440,
'downloader/request_count': 2,
'downloader/request_method_count/GET': 2,
'downloader/response_bytes': 16658,
'downloader/response_count': 2,
'downloader/response_status_count/200': 2,
'finish_reason': 'finished',
'finish_time': datetime.datetime(2016, 9, 30, 14, 24, 54, 540733),
'item_scraped_count': 1,
'log_count/DEBUG': 4,
'log_count/INFO': 7,
'response_received_count': 2,
'scheduler/dequeued': 1,
'scheduler/dequeued/memory': 1,
'scheduler/enqueued': 1,
'scheduler/enqueued/memory': 1,
'start_time': datetime.datetime(2016, 9, 30, 14, 24, 53, 674683)}
2016-09-30 22:24:54 [scrapy] INFO: Spider closed (finished)

最佳答案

月排行榜 / 总排行榜

SixPy

2016-10-3 11:58:58

'downloader/response_status_count/200': 2,

成功2次了阿

跳转到最佳答案楼层

mgiftdc

求助啊！！来个大神呀！！！

SixPy

'downloader/response_status_count/200': 2,

成功2次了阿