[已解决]爬虫抓取豆瓣读书top250的时候，无法读取页码值，试了很多种方法。。。

hanfen_aki

# 导入所需库
import requests
import bs4
import urllib3


# 访问top250首页
def open_url(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
    res = requests.get(url, headers=headers)

    return res


# 获取单页的书本名，评分，资料
def find_books(res):
    # 熬一锅汤先
    soup = bs4.BeautifulSoup(res.text, 'html.parser')

    # 找书名并保存到列表中
    name = []
    targets_name = soup.find_all('div', class_='pl2')
    for each in targets_name:
        name.append(each.a.text)

    # 评分
    rank = []
    targets_rank = soup.find_all('span', class_='rating_num')
    for each in targets_rank:
        rank.append('评分：%s' % each.text)

    # 资料
    message = []
    targets_message = soup.find_all('div', class_='pl')
    for each in targets_message:
        try:
            message.append(each.p.text)
        except:
            continue

    # 汇总信息
    result = []
    length = len(name)
    for i in range(length):
        result.append('《' + name[i] + '》' + '\n' + rank[i] + '\n'+ message[i] +'\n''\n' )

    return result


# 获取页数
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')
    depth = soup.find('span', class_='next').previous_Sibling.previous_Sibling.text

    return int(depth)


def main():
    host = 'https://book.douban.com/top250'
    res = open_url(host)
    depth = find_depth(res)

    result = []
    for i in range(depth):
        url = host + '?start=' + str(i * 25)
        res = open_url(url)
        result.extend(find_books(res))

    with open('豆瓣读书TOP250.txt', 'w', encoding='utf-8') as f:
        for each in result:
            f.write(each)


if __name__ == '__main__':
    main()

-------------------------

一直报错，无法找到text的值-。-：

depth = find_depth(res)
  File "/Users/fen/PycharmProjects/pythonProject1/findTop250book.py", line 54, in find_depth
    depth = soup.find('span', class_='next').previous_Sibling.previous_Sibling.text
AttributeError: 'NoneType' object has no attribute 'previous_Sibling'

最佳答案

月排行榜 / 总排行榜

Twilight6

2022-2-7 10:59:01

hanfen_aki 发表于 2022-2-7 10:51
谢谢大佬！！！改完之后确实可以运行了，但是
运行之后，又出现了列表范围超出的报错T。T

[b]

targets_rank = soup.find_all('span', class_='rating_num') 中 class 改成 rating_nums

targets_message = soup.find_all('div', class_='pl') 改成 soup.find_all('p', class_='pl')

for 循环中 message.append(each.p.text) 改成 message.append(each.text)

最后建议在 name[i ] 加上 .strip() 函数，参考代码：

import requests
import bs4
import urllib3


# 访问top250首页
def open_url(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
    res = requests.get(url, headers=headers)

    return res


# 获取单页的书本名，评分，资料
def find_books(res):
    # 熬一锅汤先
    soup = bs4.BeautifulSoup(res.text, 'html.parser')

    # 找书名并保存到列表中
    name = []
    targets_name = soup.find_all('div', class_='pl2')
    for each in targets_name:
        name.append(each.a.text)

    # 评分
    rank = []
    targets_rank = soup.find_all('span', class_='rating_nums')
    for each in targets_rank:
        rank.append('评分：%s' % each.text)

    # 资料
    message = []
    targets_message = soup.find_all('p', class_='pl')
    for each in targets_message:
        try:
            message.append(each.text)
        except:
            continue

    # 汇总信息
    result = []
    length = len(name)
    for i in range(length):
        result.append('《' + name[i].strip() + '》' + '\n' + rank[i] + '\n'+ message[i] +'\n''\n' )

    return result


# 获取页数
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')
    depth = soup.find('span', class_='next').previous_sibling.previous_sibling.text

    return int(depth)


def main():
    host = 'https://book.douban.com/top250'
    res = open_url(host)
    depth = find_depth(res)

    result = []
    for i in range(depth):
        url = host + '?start=' + str(i * 25)
        res = open_url(url)
        result.extend(find_books(res))

    with open('豆瓣读书TOP250.txt', 'w', encoding='utf-8') as f:
        for each in result:
            f.write(each)


if __name__ == '__main__':
    main()

[/b]

跳转到最佳答案楼层

大马强

 depth = soup.find('span', class_='next').previous_Sibling.previous_Sibling.text

两个Sibling 的s都是小写

hanfen_aki

谢谢大佬！！！改完之后确实可以运行了，但是
运行之后，又出现了列表范围超出的报错T。T

# 汇总信息
  

result = []
    length = len(name)
    for i in range(length):
        result.append('《' + name[i] + '》' + '\n' + rank[i] + '\n' + message[i] + '\n''\n')

    return result

IndexError: list index out of range

Twilight6

hanfen_aki 发表于 2022-2-7 10:51
谢谢大佬！！！改完之后确实可以运行了，但是
运行之后，又出现了列表范围超出的报错T。T

[b]

targets_rank = soup.find_all('span', class_='rating_num') 中 class 改成 rating_nums

targets_message = soup.find_all('div', class_='pl') 改成 soup.find_all('p', class_='pl')

for 循环中 message.append(each.p.text) 改成 message.append(each.text)

最后建议在 name[i ] 加上 .strip() 函数，参考代码：

import requests
import bs4
import urllib3


# 访问top250首页
def open_url(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
    res = requests.get(url, headers=headers)

    return res


# 获取单页的书本名，评分，资料
def find_books(res):
    # 熬一锅汤先
    soup = bs4.BeautifulSoup(res.text, 'html.parser')

    # 找书名并保存到列表中
    name = []
    targets_name = soup.find_all('div', class_='pl2')
    for each in targets_name:
        name.append(each.a.text)

    # 评分
    rank = []
    targets_rank = soup.find_all('span', class_='rating_nums')
    for each in targets_rank:
        rank.append('评分：%s' % each.text)

    # 资料
    message = []
    targets_message = soup.find_all('p', class_='pl')
    for each in targets_message:
        try:
            message.append(each.text)
        except:
            continue

    # 汇总信息
    result = []
    length = len(name)
    for i in range(length):
        result.append('《' + name[i].strip() + '》' + '\n' + rank[i] + '\n'+ message[i] +'\n''\n' )

    return result


# 获取页数
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')
    depth = soup.find('span', class_='next').previous_sibling.previous_sibling.text

    return int(depth)


def main():
    host = 'https://book.douban.com/top250'
    res = open_url(host)
    depth = find_depth(res)

    result = []
    for i in range(depth):
        url = host + '?start=' + str(i * 25)
        res = open_url(url)
        result.extend(find_books(res))

    with open('豆瓣读书TOP250.txt', 'w', encoding='utf-8') as f:
        for each in result:
            f.write(each)


if __name__ == '__main__':
    main()

[/b]

hanfen_aki

谢谢改写！！！

Passepartout