[已解决]低级爬虫问题

zhoujie13

import requests
from lxml import html


def spider(sn):

    url = 'https://re.taobao.com/search_ou?refpid=mm_26632258_3504122_32538762&clk1=0c5e32ca052fe01c4cf901fa6453d137&keyword={}&_input_charset=utf-8&page=0&isinner=0'.format(sn)
    html_data = requests.get(url).text
    selector = html.fromstring(html_data)

    ur_list = selector.xpath('//div[@class="item"]')
    print(len(ur_list))

    for li in ur_list:
        title = li.xpath('div[@class="info"]/span/@title')
        print(title)


if __name__ == '__main__':
    sn = '9787115428028'
    spider(sn)

不知道哪里出错了，爬到的都是空的

最佳答案

月排行榜 / 总排行榜

XiaoPaiShen

2019-10-7 12:03:08

zhoujie13 发表于 2019-10-7 11:21
用requests库怎么解决呢

import requests
from lxml import html


def spider(sn):

    url = 'https://re.taobao.com/search_ou?refpid=mm_26632258_3504122_32538762&clk1=0c5e32ca052fe01c4cf901fa6453d137&keyword={}&_input_charset=utf-8&page=0&isinner=0'.format(sn)
    html_data = requests.get(url).text
    selector = html.fromstring(html_data)

    titles = selector.xpath('//div[@class="item"]/a/div[@class="info"]/span/@title')
    
    for title in titles:        
        print(title)


if __name__ == '__main__':
    sn = '9787115428028'
    spider(sn)

跳转到最佳答案楼层

wp231957

淘宝都有反爬吧

zhoujie13

wp231957 发表于 2019-10-5 08:59
淘宝都有反爬吧

不知道，看视频他是可以的

geen

zhoujie13 发表于 2019-10-5 09:41
不知道，看视频他是可以的

对方使用了代理池吗？

XiaoPaiShen

import urllib.request
from lxml import html


def spider(sn):

    url = 'https://re.taobao.com/search_ou?refpid=mm_26632258_3504122_32538762&clk1=0c5e32ca052fe01c4cf901fa6453d137&keyword={}&_input_charset=utf-8&page=0&isinner=0'.format(sn)
    response = urllib.request.urlopen(url)
    html_data = response.read().decode('utf-8')    
    
    selector = html.fromstring(html_data)

    titles = selector.xpath('//div[@class="item"]/a/div[@class="info"]/span/@title')
    
    for title in titles:        
        print(title)
        


if __name__ == '__main__':
    sn = '9787115428028'
    spider(sn)

zhoujie13

geen 发表于 2019-10-7 09:36
对方使用了代理池吗？

没有吧，很简单的代码

zhoujie13

XiaoPaiShen 发表于 2019-10-7 10:41

用requests库怎么解决呢

XiaoPaiShen

zhoujie13 发表于 2019-10-7 11:21
用requests库怎么解决呢

import requests
from lxml import html


def spider(sn):

    url = 'https://re.taobao.com/search_ou?refpid=mm_26632258_3504122_32538762&clk1=0c5e32ca052fe01c4cf901fa6453d137&keyword={}&_input_charset=utf-8&page=0&isinner=0'.format(sn)
    html_data = requests.get(url).text
    selector = html.fromstring(html_data)

    titles = selector.xpath('//div[@class="item"]/a/div[@class="info"]/span/@title')
    
    for title in titles:        
        print(title)


if __name__ == '__main__':
    sn = '9787115428028'
    spider(sn)