[已解决]爬取网站标题，作者信息，简介，文章内容

Thewhitecrow

1. 
   
2. from urllib.request import urlopen, Request
   
3. from bs4 import BeautifulSoup  # 网页解析  获取数据
   
4. import re  # 正则表达式 进行文字匹配
   
5. import random
   
6. 
   
7. url = 'https://cybernews.com/news/'
   
8. headers = {'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'}
   
9. ret = Request(url)
   
10. html = urlopen(ret)
    
11. bs = BeautifulSoup(html, 'html.parser')
    
12. 
    
13. names = bs.find_all("h3",{"class":"jeg_post_title"})
    
14. for name in names:
    
15.     name = name.get_text()
    
16.     print(name)
    
17. neirongs = bs.find_all("div",{"class":"jeg_post_meta"})
    
18. for neirong in neirongs:
    
19.     neirong = neirong.get_text()
    
20.     print(neirong)
    
21. contents = bs.find_all("div",{"class":"jeg_post_excerpt"})
    
22. for content in contents:
    
23.     content = content.get_text()
    
24.     print(content)
    
25. 
    

复制代码

刚接触爬虫，大佬们能看下问题所在吗，以及爬取文章内容这个不太会

最佳答案

月排行榜 / 总排行榜

笨鸟学飞

2021-2-1 20:02:21

1. import requests
   
2. from lxml import etree
   
3. 
   
4. 
   
5. if __name__ == '__main__':
   
6.     url = 'https://cybernews.com/news/'
   
7.     headers = {
   
8.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}
   
9.     res = requests.get(url, headers=headers)
   
10.     res.encoding = res.apparent_encoding
    
11.     tree = etree.HTML(res.text)
    
12.     # 标题
    
13.     titles = tree.xpath('//div[@class="jeg_posts jeg_load_more_flag"]/article/h3/a/text()')
    
14.     # 作者
    
15.     authors = tree.xpath('//div[@class="jeg_meta_author"]/a/text()')
    
16.     # 简介
    
17.     excerpts = tree.xpath('//div[@class="jeg_post_excerpt"]/p/text()')
    
18.     # 详情页面url
    
19.     text_urls = tree.xpath('//div[@class="jeg_posts jeg_load_more_flag"]/article/h3/a/@href')
    
20.     with open('./1.txt', 'w', encoding='utf-8') as fp:
    
21.         for title,anthor,excerpt,text_url in zip(titles, authors, excerpts, text_urls):
    
22.             fp.write(title+'\nby:'+anthor+'\n'+excerpt+'\n'+text_url+'\n\n')

复制代码

==============
你的headers写法是错误的，少了冒号。。。
详情页面的数据爬取自己写吧，已经到了这个地步了相信难不倒你了

跳转到最佳答案楼层

YunGuo

定义了headers不添加进去？headers是一个集合？响应结果不读取？建议基础多学学再动手
这里改下

1. headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'}
   
2. ret = Request(url, headers=headers)
   
3. html = urlopen(ret)
   
4. bs = BeautifulSoup(html.read(), 'html.parser')

复制代码

私はり

这三个内容不用分开使用三次循环 太麻烦了

笨鸟学飞

1. import requests
   
2. from lxml import etree
   
3. 
   
4. 
   
5. if __name__ == '__main__':
   
6.     url = 'https://cybernews.com/news/'
   
7.     headers = {
   
8.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}
   
9.     res = requests.get(url, headers=headers)
   
10.     res.encoding = res.apparent_encoding
    
11.     tree = etree.HTML(res.text)
    
12.     # 标题
    
13.     titles = tree.xpath('//div[@class="jeg_posts jeg_load_more_flag"]/article/h3/a/text()')
    
14.     # 作者
    
15.     authors = tree.xpath('//div[@class="jeg_meta_author"]/a/text()')
    
16.     # 简介
    
17.     excerpts = tree.xpath('//div[@class="jeg_post_excerpt"]/p/text()')
    
18.     # 详情页面url
    
19.     text_urls = tree.xpath('//div[@class="jeg_posts jeg_load_more_flag"]/article/h3/a/@href')
    
20.     with open('./1.txt', 'w', encoding='utf-8') as fp:
    
21.         for title,anthor,excerpt,text_url in zip(titles, authors, excerpts, text_urls):
    
22.             fp.write(title+'\nby:'+anthor+'\n'+excerpt+'\n'+text_url+'\n\n')

复制代码

==============
你的headers写法是错误的，少了冒号。。。
详情页面的数据爬取自己写吧，已经到了这个地步了相信难不倒你了