[已解决]低级爬虫问题

zhoujie13

import requests
from lxml import html


def spider(sn):

    url = 'https://re.taobao.com/search_ou?refpid=mm_26632258_3504122_32538762&clk1=0c5e32ca052fe01c4cf901fa6453d137&keyword={}&_input_charset=utf-8&page=0&isinner=0'.format(sn)
    html_data = requests.get(url).text
    selector = html.fromstring(html_data)

    ur_list = selector.xpath('//div[@class="item"]')
    print(len(ur_list))

    for li in ur_list:
        title = li.xpath('div[@class="info"]/span/@title')
        print(title)


if __name__ == '__main__':
    sn = '9787115428028'
    spider(sn)

不知道哪里出错了，爬到的都是空的

最佳答案

月排行榜 / 总排行榜

XiaoPaiShen

2019-10-7 12:03:08

zhoujie13 发表于 2019-10-7 11:21
用requests库怎么解决呢

1. import requests
   
2. from lxml import html
   
3. 
   
4. 
   
5. def spider(sn):
   
6. 
   
7.     url = 'https://re.taobao.com/search_ou?refpid=mm_26632258_3504122_32538762&clk1=0c5e32ca052fe01c4cf901fa6453d137&keyword={}&_input_charset=utf-8&page=0&isinner=0'.format(sn)
   
8.     html_data = requests.get(url).text
   
9.     selector = html.fromstring(html_data)
   
10. 
    
11.     titles = selector.xpath('//div[@class="item"]/a/div[@class="info"]/span/@title')
    
12.    
    
13.     for title in titles:        
    
14.         print(title)
    
15. 
    
16. 
    
17. if __name__ == '__main__':
    
18.     sn = '9787115428028'
    
19.     spider(sn)

复制代码

跳转到最佳答案楼层

wp231957

淘宝都有反爬吧

zhoujie13

wp231957 发表于 2019-10-5 08:59
淘宝都有反爬吧

不知道，看视频他是可以的

geen

zhoujie13 发表于 2019-10-5 09:41
不知道，看视频他是可以的

对方使用了代理池吗？

XiaoPaiShen

1. import urllib.request
   
2. from lxml import html
   
3. 
   
4. 
   
5. def spider(sn):
   
6. 
   
7.     url = 'https://re.taobao.com/search_ou?refpid=mm_26632258_3504122_32538762&clk1=0c5e32ca052fe01c4cf901fa6453d137&keyword={}&_input_charset=utf-8&page=0&isinner=0'.format(sn)
   
8.     response = urllib.request.urlopen(url)
   
9.     html_data = response.read().decode('utf-8')   
   
10.    
    
11.     selector = html.fromstring(html_data)
    
12. 
    
13.     titles = selector.xpath('//div[@class="item"]/a/div[@class="info"]/span/@title')
    
14.    
    
15.     for title in titles:        
    
16.         print(title)
    
17.         
    
18. 
    
19. 
    
20. if __name__ == '__main__':
    
21.     sn = '9787115428028'
    
22.     spider(sn)

复制代码

zhoujie13

geen 发表于 2019-10-7 09:36
对方使用了代理池吗？

没有吧，很简单的代码

zhoujie13

XiaoPaiShen 发表于 2019-10-7 10:41

用requests库怎么解决呢

XiaoPaiShen

zhoujie13 发表于 2019-10-7 11:21
用requests库怎么解决呢

1. import requests
   
2. from lxml import html
   
3. 
   
4. 
   
5. def spider(sn):
   
6. 
   
7.     url = 'https://re.taobao.com/search_ou?refpid=mm_26632258_3504122_32538762&clk1=0c5e32ca052fe01c4cf901fa6453d137&keyword={}&_input_charset=utf-8&page=0&isinner=0'.format(sn)
   
8.     html_data = requests.get(url).text
   
9.     selector = html.fromstring(html_data)
   
10. 
    
11.     titles = selector.xpath('//div[@class="item"]/a/div[@class="info"]/span/@title')
    
12.    
    
13.     for title in titles:        
    
14.         print(title)
    
15. 
    
16. 
    
17. if __name__ == '__main__':
    
18.     sn = '9787115428028'
    
19.     spider(sn)

复制代码