豆瓣电影爬取前200条评论
代码比较简单，不做注释了import requests,json,bs4
from bs4 import BeautifulSoup
q=1
for page in range(0,200,20):
url= "https://movie.douban.com/subject/35376457/comments?percent_type=&start={}&limit=20&status=P".format(page)
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0'}
res=requests.get(url=url,headers=headers)
#s = json.loads(res.text)
soup = BeautifulSoup(res.text, 'html.parser')
for each in soup.select('div'):
s1 = each.select('span').get_text().replace('\n','').replace(' ','')[:10]
s2 = each.select('span').get_text()
s3 = each.select('span').get_text().split('\n')
s4 = each.select('span').get_text().replace('\n','')
s5 = each.select('span').select('span').attrs['title']
print(q,s5,s1,s2,s3,s4)
q += 1 运行结果 {:5_95:}
页:
[1]