爬取豆瓣电影top250信息

不2b的铅笔

明天再拿其他网站练练手

'''爬取豆瓣电影top250'''

#导入所需库
import requests
import bs4

#访问top250首页
def open_url(url):
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
    res = requests.get(url,headers = headers)

    return res

#获取单页的电影名，评分，资料
def find_movies(res):
    #熬一锅汤先
    soup = bs4.BeautifulSoup(res.text,'html.parser')

    #找电影名并保存到列表中
    name = []
    targets_name = soup.find_all('div',class_='hd')
    for each in targets_name:
        name.append(each.a.span.text)

    #评分
    rank = []
    targets_rank = soup.find_all('span',class_='rating_num')
    for each in targets_rank:
        rank.append('评分：%s' %each.text)

    #资料
    message = []
    targets_message = soup.find_all('div',class_='bd')
    for each in targets_message:
        try:
            message.append(each.p.text.split('\n')[1].strip() + each.p.text.split('\n')[2].strip())
        except:
            continue

    #汇总信息
    result = []
    length = len(name)
    for i in range(length):
        result.append('《' + name[i] + '》' + '\n' + rank[i] + '\n' + message[i] + '\n\n')

    return result


#获取页数
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')
    depth = soup.find('a',href='?start=225&filter=').text

    return int(depth)

def main():
    host = 'http://movie.douban.com/top250'
    res = open_url(host)
    depth = find_depth(res)

    result = []
    for i in range(depth):
        url = host + '?start=' + str(i*25) + '&filter='
        res = open_url(url)
        result.extend(find_movies(res))

    with open('豆瓣电影TOP250.txt','w',encoding='utf-8') as f:
        for each in result:
            f.write(each)

if __name__=='__main__':
    main()

结果如图

hornwong

hanfen_aki

优秀的！！！

隔壁老程呀

厉害了