|
马上注册,结交更多好友,享用更多功能^_^
您需要 登录 才可以下载或查看,没有账号?立即注册
x
写的代码只能爬取网站第一组数据,后面的爬不了。@不二如是
代码:
import scrapy
from tutorial.items import DmozItem
class DmozSpidera(scrapy.Spider):
name = 'dmoz'
allowed_domains = ['dmoz-odp.org']
start_urls = [
'https://www.dmoz-odp.org/Computers/Programming/Languages/Python/Books/',
'https://www.dmoz-odp.org/Computers/Programming/Languages/Python/Resources/'
]
def parse(self,response):
res = scrapy.selector.Selector(response)
i =1
sites = res.xpath('//*[@id="site-list-content"]')
items = []
for site in sites:
item = DmozItem()
ti = 'div[%d]/div[3]/a/div' % i
li = 'div[%d]/div[3]/a' % i
de = 'div[%d]/div[3]/div' % i
item['title'] = site.xpath(ti).extract()
item['link'] = site.xpath(li).extract()
item['desc'] = site.xpath(de).extract()
items.append(item)
i += 1
return items
|
|