新浪新闻首页新闻标题和链接

wcq15759797758

很简单的爬虫

1. import requests
   
2. import cchardet
   
3. import traceback
   
4. from lxml import etree
   
5. 
   
6. def downloader(url, timeout=10, headers=None, debug=False, binary=False):
   
7.     _headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'}
   
8.     redirected_url = url
   
9.     if headers:
   
10.         _headers = headers
    
11.     try:
    
12.         r = requests.get(url, headers=_headers, timeout=timeout)
    
13.         if binary:
    
14.             html = r.content
    
15.         else:
    
16.             encoding = cchardet.detect(r.content)['encoding']
    
17.             html = r.content.decode(encoding)
    
18.         status = r.status_code
    
19.         redirected_url = r.url
    
20.     except:
    
21.         if debug:
    
22.             traceback.print_exc()
    
23.         msg = 'failed download: {}'.format(url)
    
24.         print(msg)
    
25.         if binary:
    
26.             html = b''
    
27.         else:
    
28.             html = ''
    
29.         status = 0
    
30.     return title(html)
    
31. 
    
32. def title(html):
    
33.     title_html = etree.HTML(html)
    
34.     titles = title_html.xpath('//a[@target="_blank"]')
    
35.     for title in titles:
    
36.         item = {}
    
37.         tit = title.xpath('./text()')
    
38.         urls = title.xpath('./@href')
    
39.         item['title'] = str(processing(tit))
    
40.         item['url'] = str(processing(urls))
    
41.         if len(item['title']) > 4 :
    
42.                 print(item)
    
43. 
    
44. def processing(strs):
    
45.     s = ''  # 定义保存内容的字符串
    
46.     for n in strs:
    
47.         n = ''.join(n.split())  # 去除空字符
    
48.         s = s + n  # 拼接字符串
    
49.     return s      # 返回拼接后的字符串
    
50.         
    
51. 
    
52. if __name__ == '__main__':
    
53.     url = 'https://news.sina.com.cn/'
    
54.     downloader(url)

复制代码

wxm1324

学习了！

yobdc

学习

Rebecca2021

学习

qq1151985918

Rosy7673

支持一下

瑜子肌昂

鱼币选手

hornwong

937135952

学习一下

llc2009

感谢

让记忆定格

厉害了