爬虫抓取豆瓣读书top250的时候，无法读取页码值，试了很多种方法。。。
# 导入所需库
import requests
import bs4
import urllib3
# 访问top250首页
def open_url(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
res = requests.get(url, headers=headers)
return res
# 获取单页的书本名，评分，资料
def find_books(res):
# 熬一锅汤先
soup = bs4.BeautifulSoup(res.text, 'html.parser')
# 找书名并保存到列表中
name = []
targets_name = soup.find_all('div', class_='pl2')
for each in targets_name:
name.append(each.a.text)
# 评分
rank = []
targets_rank = soup.find_all('span', class_='rating_num')
for each in targets_rank:
rank.append('评分：%s' % each.text)
# 资料
message = []
targets_message = soup.find_all('div', class_='pl')
for each in targets_message:
try:
message.append(each.p.text)
except:
continue
# 汇总信息
result = []
length = len(name)
for i in range(length):
result.append('《' + name + '》' + '\n' + rank + '\n'+ message +'\n''\n' )
return result
# 获取页数
def find_depth(res):
soup = bs4.BeautifulSoup(res.text,'html.parser')
depth = soup.find('span', class_='next').previous_Sibling.previous_Sibling.text
return int(depth)
def main():
host = 'https://book.douban.com/top250'
res = open_url(host)
depth = find_depth(res)
result = []
for i in range(depth):
url = host + '?start=' + str(i * 25)
res = open_url(url)
result.extend(find_books(res))
with open('豆瓣读书TOP250.txt', 'w', encoding='utf-8') as f:
for each in result:
f.write(each)
if __name__ == '__main__':
main()
-------------------------
一直报错，无法找到text的值-。-：
depth = find_depth(res)
File "/Users/fen/PycharmProjects/pythonProject1/findTop250book.py", line 54, in find_depth
depth = soup.find('span', class_='next').previous_Sibling.previous_Sibling.text
AttributeError: 'NoneType' object has no attribute 'previous_Sibling' depth = soup.find('span', class_='next').previous_Sibling.previous_Sibling.text
两个Sibling 的s都是小写 谢谢大佬！！！改完之后确实可以运行了，但是
运行之后，又出现了列表范围超出的报错T。T
# 汇总信息
result = []
length = len(name)
for i in range(length):
result.append('《' + name + '》' + '\n' + rank + '\n' + message + '\n''\n')
return result
IndexError: list index out of range hanfen_aki 发表于 2022-2-7 10:51
谢谢大佬！！！改完之后确实可以运行了，但是
运行之后，又出现了列表范围超出的报错T。T
targets_rank = soup.find_all('span', class_='rating_num') 中 class 改成 rating_nums
targets_message = soup.find_all('div', class_='pl') 改成 soup.find_all('p', class_='pl')
for 循环中 message.append(each.p.text) 改成 message.append(each.text)
最后建议在 name 加上 .strip() 函数，参考代码：
import requests
import bs4
import urllib3
# 访问top250首页
def open_url(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
res = requests.get(url, headers=headers)
return res
# 获取单页的书本名，评分，资料
def find_books(res):
# 熬一锅汤先
soup = bs4.BeautifulSoup(res.text, 'html.parser')
# 找书名并保存到列表中
name = []
targets_name = soup.find_all('div', class_='pl2')
for each in targets_name:
name.append(each.a.text)
# 评分
rank = []
targets_rank = soup.find_all('span', class_='rating_nums')
for each in targets_rank:
rank.append('评分：%s' % each.text)
# 资料
message = []
targets_message = soup.find_all('p', class_='pl')
for each in targets_message:
try:
message.append(each.text)
except:
continue
# 汇总信息
result = []
length = len(name)
for i in range(length):
result.append('《' + name.strip() + '》' + '\n' + rank + '\n'+ message +'\n''\n' )
return result
# 获取页数
def find_depth(res):
soup = bs4.BeautifulSoup(res.text,'html.parser')
depth = soup.find('span', class_='next').previous_sibling.previous_sibling.text
return int(depth)
def main():
host = 'https://book.douban.com/top250'
res = open_url(host)
depth = find_depth(res)
result = []
for i in range(depth):
url = host + '?start=' + str(i * 25)
res = open_url(url)
result.extend(find_books(res))
with open('豆瓣读书TOP250.txt', 'w', encoding='utf-8') as f:
for each in result:
f.write(each)
if __name__ == '__main__':
main() 谢谢改写！！！ {:10_277:}
页:
[1]