马上注册,结交更多好友,享用更多功能^_^
您需要 登录 才可以下载或查看,没有账号?立即注册
x
import scrapy
from baidutieba.items import BaidutiebaItem
class Baidutieba1Spider(scrapy.Spider):
name = 'baidutieba1'
#allowed_domains = ['www.baidu.com']
start_urls = ['https://tieba.baidu.com/f?kw=python&ie=utf-8&pn=0']
def parse(self,response):
#分组
tr_list = response.xpath('//div[@id="pagelet_frs-list/pagelet/thread_list"]//li')
for tr in tr_list:
item = BaidutiebaItem()
item["title"] = tr.xpath('./div/div[2]/div[1]/div[1]/a/text()').extract_first()
item['href'] = tr.xpath('./div/div[2]/div[1]/div[1]/a/@href').extract_first()
item['publish_date'] = tr.xpath('./div/div[2]/div[2]/div[2]/span[1]/a').extract_first()
yield scrapy.Request(
item["href"],
callback=self.patse_detail,
meta = {"item":item}
)
def patse_detail(self,response):
item = response.meta["item"]
print(item)
|