[已解决]爬取豆瓣 电影 标题问题

费小牛

请问如何获取审查元素的电影名称，  我怎么是获得一个空列表？哪里错了了吗

最佳答案

月排行榜 / 总排行榜

Twilight6

2020-6-3 07:45:36

费小牛 发表于 2020-6-2 22:07

你看3L 啊  被无视的赶脚有点不舒服  这样改就可以了

豆瓣有的时候爬到的名称 不是在 a.span里  有的就在a里要加个判断  看代码吧

import requests
from bs4 import BeautifulSoup
import re


def open_url(url):
    hearders = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
    res = requests.get(url, headers=hearders)
    return res


def find_depth(res):
    pass


def biaoti(res):
    # 电影名字
    movies = []
    req = BeautifulSoup(res.text, 'html.parser')
    name = req.find_all('div', class_=['pl2'])
    for each1 in name:
        if not (each1.a.span):
            movies.append((each1.a.text).strip())
            continue
        movies.append(each1.a.span.text)
    print(movies)

    # 评分
    ranks = []
    rank = req.find_all('span', class_="rating_nums")
    for each2 in rank:
        ranks.append(each2.text)
    print(ranks)
    # 内容
    messages = []
    message = req.find_all("p", class_=['pl'])
    for each3 in message:
        messages.append(each3.text)

    result = []
    moveies_num = len(movies)
    for i in range(moveies_num):
        title = movies[i] + '->' + ranks[i] + '->' + messages[i] + '\n'
        result.append(title)
    print(result)
    return result


def main():
    url = 'https://movie.douban.com/chart'
    res = open_url(url)
    nr = biaoti(res)
    with open('豆瓣本周排行名单.txt','w',encoding='utf-8') as f:
        for a in nr:
            f.write(a+'\n')
main()

跳转到最佳答案楼层

suchocolate

发下代码或网页url

Twilight6

改成这样：

req.find_all('div',class_=['pl2'])

费小牛

suchocolate 发表于 2020-6-2 20:49
发下代码或网页url

import requests
from bs4 import BeautifulSoup
import re

def open_url(url):
    hearders = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
    res = requests.get(url,headers=hearders)
    return res

def find_depth(res):
    pass

def biaoti(res):
    #电影名字
    movies = []
    req = BeautifulSoup(res.text,'html.parser')
    name = req.find_all('div',class_=['p12'])
    for each1 in name:
        movies.append(each1.a.span.text)
    print(movies)

    #评分
    ranks =[]
    rank = req.find_all('span',class_="rating_nums")
    for each2 in rank:
        ranks.append(each2.text)
    print(ranks)
    #内容
    messages =[]
    message = req.find_all("p", class_="pl")
    for each3 in message:
        messages.append(each3.text)

    result =[]
    moveies_num = len(movies)
    for i in range(moveies_num):
        title = movies[i]+'->'+ranks[i]+'->'+messages[i]+'\n'
        result.append(title)
    print(result)
    return result

def main():
    url = 'https://movie.douban.com/chart'
    res = open_url(url)
    nr = biaoti(res)
    with open('豆瓣本周排行名单.txt','w') as f:
        for a in nr:
            f.write(nr)
        

听风夜雨

你爬不爬的出来我不知道，但是div的class是等于pl2不是p12。如果你刚学爬虫建议写一点运行一点，不要一股脑写完

Tip0

费小牛 发表于 2020-6-2 22:07

你代码错了，3l的是['pl2'] 你的是p12

Twilight6

费小牛 发表于 2020-6-2 22:07

你看3L 啊  被无视的赶脚有点不舒服  这样改就可以了

豆瓣有的时候爬到的名称 不是在 a.span里  有的就在a里要加个判断  看代码吧

import requests
from bs4 import BeautifulSoup
import re


def open_url(url):
    hearders = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
    res = requests.get(url, headers=hearders)
    return res


def find_depth(res):
    pass


def biaoti(res):
    # 电影名字
    movies = []
    req = BeautifulSoup(res.text, 'html.parser')
    name = req.find_all('div', class_=['pl2'])
    for each1 in name:
        if not (each1.a.span):
            movies.append((each1.a.text).strip())
            continue
        movies.append(each1.a.span.text)
    print(movies)

    # 评分
    ranks = []
    rank = req.find_all('span', class_="rating_nums")
    for each2 in rank:
        ranks.append(each2.text)
    print(ranks)
    # 内容
    messages = []
    message = req.find_all("p", class_=['pl'])
    for each3 in message:
        messages.append(each3.text)

    result = []
    moveies_num = len(movies)
    for i in range(moveies_num):
        title = movies[i] + '->' + ranks[i] + '->' + messages[i] + '\n'
        result.append(title)
    print(result)
    return result


def main():
    url = 'https://movie.douban.com/chart'
    res = open_url(url)
    nr = biaoti(res)
    with open('豆瓣本周排行名单.txt','w',encoding='utf-8') as f:
        for a in nr:
            f.write(a+'\n')
main()

费小牛

Twilight6 发表于 2020-6-3 07:45
你看3L 啊  被无视的赶脚有点不舒服  这样改就可以了

豆瓣有的时候爬到的名称 不是在 a.span里  有的 ...

我没有无视的，我觉得我没有把问题说清楚，索性直接上代码了。感谢大佬的细心解释，我代码中的还有很多错误 也一并帮我修正了 。同时也感谢其他大佬对我的帮助

Twilight6

费小牛 发表于 2020-6-3 12:41
我没有无视的，我觉得我没有把问题说清楚，索性直接上代码了。感谢大佬的细心解释，我代码中的 ...

没事  加油