爬取豆瓣电影top250信息

不2b的铅笔

明天再拿其他网站练练手

1. '''爬取豆瓣电影top250'''
   
2. 
   
3. #导入所需库
   
4. import requests
   
5. import bs4
   
6. 
   
7. #访问top250首页
   
8. def open_url(url):
   
9.     headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
   
10.     res = requests.get(url,headers = headers)
    
11. 
    
12.     return res
    
13. 
    
14. #获取单页的电影名，评分，资料
    
15. def find_movies(res):
    
16.     #熬一锅汤先
    
17.     soup = bs4.BeautifulSoup(res.text,'html.parser')
    
18. 
    
19.     #找电影名并保存到列表中
    
20.     name = []
    
21.     targets_name = soup.find_all('div',class_='hd')
    
22.     for each in targets_name:
    
23.         name.append(each.a.span.text)
    
24. 
    
25.     #评分
    
26.     rank = []
    
27.     targets_rank = soup.find_all('span',class_='rating_num')
    
28.     for each in targets_rank:
    
29.         rank.append('评分：%s' %each.text)
    
30. 
    
31.     #资料
    
32.     message = []
    
33.     targets_message = soup.find_all('div',class_='bd')
    
34.     for each in targets_message:
    
35.         try:
    
36.             message.append(each.p.text.split('\n')[1].strip() + each.p.text.split('\n')[2].strip())
    
37.         except:
    
38.             continue
    
39. 
    
40.     #汇总信息
    
41.     result = []
    
42.     length = len(name)
    
43.     for i in range(length):
    
44.         result.append('《' + name[i] + '》' + '\n' + rank[i] + '\n' + message[i] + '\n\n')
    
45. 
    
46.     return result
    
47. 
    
48. 
    
49. #获取页数
    
50. def find_depth(res):
    
51.     soup = bs4.BeautifulSoup(res.text,'html.parser')
    
52.     depth = soup.find('a',href='?start=225&filter=').text
    
53. 
    
54.     return int(depth)
    
55. 
    
56. def main():
    
57.     host = 'http://movie.douban.com/top250'
    
58.     res = open_url(host)
    
59.     depth = find_depth(res)
    
60. 
    
61.     result = []
    
62.     for i in range(depth):
    
63.         url = host + '?start=' + str(i*25) + '&filter='
    
64.         res = open_url(url)
    
65.         result.extend(find_movies(res))
    
66. 
    
67.     with open('豆瓣电影TOP250.txt','w',encoding='utf-8') as f:
    
68.         for each in result:
    
69.             f.write(each)
    
70. 
    
71. if __name__=='__main__':
    
72.     main()
    

复制代码

结果如图

hornwong

hanfen_aki

优秀的！！！

隔壁老程呀

厉害了