豆瓣电影爬取前200条评论

孟大大

代码比较简单，不做注释了

1. import requests,json,bs4
   
2. from bs4 import BeautifulSoup
   
3. q=1
   
4. for page in range(0,200,20):
   
5.     url= "https://movie.douban.com/subject/35376457/comments?percent_type=&start={}&limit=20&status=P".format(page)
   
6.     headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0'}
   
7.     res=requests.get(url=url,headers=headers)
   
8.     #s = json.loads(res.text)
   
9.     soup = BeautifulSoup(res.text, 'html.parser')
   
10.     for each in soup.select('div[class="comment"]'):
    
11.         s1 = each.select('span[class="comment-time"]')[0].get_text().replace('\n','').replace(' ','')[:10]
    
12.         s2 = each.select('span[class="votes vote-count"]')[0].get_text()
    
13.         s3 = each.select('span[class="comment-info"]')[0].get_text().split('\n')[1]
    
14.         s4 = each.select('span[class="short"]')[0].get_text().replace('\n','')
    
15.         s5 = each.select('span[class="comment-info"]')[0].select('span')[1].attrs['title']
    
16.         print(q,s5,s1,s2,s3,s4)
    
17.         q += 1

复制代码

孟大大

运行结果

hornwong