爬取新浪微博，实时，关键词搜索内容
本帖最后由 疾风怪盗 于 2020-9-9 21:32 编辑最近又学习了下爬虫的视频，学习了json的数据格式爬取，把之前做的获取‘松江’为关键词的实施新浪微博的代码重新做了一遍
发现这是不是就不用cookie了，直接就可以获取？会被封IP么？还是会怎么反爬？
import requests, json, time
class XinlangWeibo():
def __init__(self, name):
self.url = f'https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D61%26q={name}&page_type=searchall&page='
self.headers = {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1'}
self.data = []
def Get_Data(self, page):# 主程序，爬取数据
for num in range(1, page):
url = self.url + str(num)
print(url)
response = requests.get(url=url, headers=self.headers)
data_all = json.loads(response.content.decode('utf-8'))
data_all = data_all['data']['cards']
for i in range(len(data_all)):
if data_all['card_type'] == 9:
data_temp = {}
data_temp['screen_name'] = data_all['mblog']['user']['screen_name']# 发帖人昵称
data_temp['screen_name'] = data_temp['screen_name'] if len(data_temp['screen_name']) > 0 else None
data_temp['profile_url'] = data_all['mblog']['user']['profile_url']# 发帖人主页
data_temp['profile_url'] = data_temp['profile_url'] if len(data_temp['profile_url']) > 0 else None
data_temp['created_time'] = data_all['mblog']['created_at']# 发帖时间距离现在时间
data_temp['created_time'] = data_temp['created_time'] if len(
data_temp['created_time']) > 0 else None
data_temp['raw_text'] = data_all['mblog']['raw_text']# 发帖内容
data_temp['raw_text'] = data_temp['raw_text'] if len(data_temp['raw_text']) > 0 else None
data_temp['scheme_url'] = data_all['scheme']# 帖子地址
data_temp['scheme_url'] = data_temp['scheme_url'] if len(data_temp['scheme_url']) > 0 else None
self.data.append(data_temp)
time.sleep(3)
return data_all
def Save_Josn(self, data_all):
with open('新浪微博-dataall.json', 'w', encoding='utf-8') as files:# 保存json文件
json.dump(data_all, files, ensure_ascii=False, indent=4)
def Save_Data(self):
with open('新浪微博-data.txt', 'w', encoding='utf-8') as file:# 保存帖子内容
for s in range(0, len(self.data)):
file.write(f"第{s + 1}个帖子为："
+ "\n" + f"微博发帖人为：{self.data['screen_name']}"
+ "\n" + f"微博发帖人主页为：{self.data['profile_url']}"
+ "\n" + f"发帖时间为：{self.data['created_time']}"
+ "\n" + f"帖子内容为：{self.data['raw_text']}"
+ "\n" + f"帖子链接为：{self.data['scheme_url']}"
+ "\n" + "*" * 100 + "\n"
+ "\n")
print('已保存')
if __name__ == '__main__':
xinlangweibo = XinlangWeibo(name='松江')# 关键词
data_all = xinlangweibo.Get_Data(page=3)# 获取页数
# xinlangweibo.Save_Josn(data_all)
xinlangweibo.Save_Data()
学习学习。。。。。。 学习一下 来，学习一下
{:5_102:} 111111111111111111111111111111111111111111111 看看 {:10_277:} 封IP看网易的心情吧 学习 看看 学习了 厉害嘞 🐮啊 666 {:5_95:}
666
really？
666 ！