import requests
from bs4 import BeautifulSoup
url = 'https://www.shicimingju.com/book/sanguoyanyi.html'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
# 以下为修改的代码 该网页是用utf-8 来编码,所以要进行编码处理,这是第一处
req = requests.get(url=url, headers=headers)
req.encoding = ("utf-8")
page_text = req.text
fp = open('./sanguo.txt', 'w', encoding='utf-8')
soup = BeautifulSoup(page_text, 'lxml')
a_list = soup.select('.book-mulu > ul > li > a')
for a in a_list:
title = a.string
detail_url = 'https://www.shicimingju.com'+a['href']
# 这是第二处
req2 = requests.get(url=detail_url, headers=headers)
req2.encoding = ("utf-8")
page_text_detail = req2.text
soup = BeautifulSoup(page_text_detail, 'lxml')
div_tag = soup.find('div', class_="chapter_content")
content = div_tag.text
fp.write(title+':'+content+'\n')
print(title, '保存成功!!!')
fp.close()
乱码一般都是你的编码有问题,参考这篇文章
https://blog.csdn.net/wyb199026/ ... 1018.2226.3001.4187 |