马上注册，结交更多好友，享用更多功能^_^
您需要 登录 才可以下载或查看，没有账号？立即注册
x
import requests
import bs4
import re
def open_url(url):
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
res = requests.get(url = url,headers=headers)
return res
def find_title(res):
soup = bs4.BeautifulSoup(res.text,"html.parser")
title = []
targets = soup.find_all("div",class_= "WB_text W_f14")
for each in targets:
title.append(each.a.text)
result = []
length = len(title)
for i in range(length):
result.append(title[i]+ '\n')
return result
def find_depth(res):
soup = bs4.BeautifulSoup(res.text, 'html.parser')
depth = soup.find('a', class_='page next S_txt1 S_line1').previous_sibling.previous_sibling.text
return int(depth)
def main():
host = "https://weibo.com/rmrb?is_all=1&stat_date=202001&page=1#feedtop"
res = open_url(host)
depth = find_depth(res)
result = []
for i in range(depth):
url = "https://weibo.com/rmrb?is_all=1&stat_date=202001&page="+ str(i+1)+"#feedtop"
res = open_url(url)
result.extend(find_title(res))
with open("新闻标题.txt", "w", encoding="utf-8") as f:
for each in result:
f.write(each)
if __name__ == "__main__":
main()
想翻页爬取微博标题，但是运行提示我报错，要怎么修改呢，soup没怎么学会
| 
( 粤ICP备18085999号-1 | 粤公网安备 44051102000585号)
狗仔卡
发表于 2020-9-9 10:06:01
置顶卡
千斤顶
显身卡
发表于 2020-9-9 10:19:15
楼主