微博评论内容爬取

underwood_yo

同学需要爬取微博的评论内容以及时间，给了一个excel文件，包含需要爬取的网址：

                               
登录/注册后可看大图

1. 
   
2. import numpy as np
   
3. import pandas as pd
   
4. from selenium import webdriver
   
5. import re
   
6. from selenium.webdriver.common.by import By
   
7. from bs4 import BeautifulSoup
   
8. import time
   
9. 
   
10. chrome_options = webdriver.ChromeOptions()
    
11. chrome_options.add_argument('--headless')
    
12. browser = webdriver.Chrome(options=chrome_options)
    
13. 
    
14. start_time = time.time()
    
15. 
    
16. def crawer(url):
    
17.     print('开始爬取'+ url)
    
18.     browser.get(url)
    
19.     time.sleep(1)
    
20.     res = browser.page_source
    
21.     return res
    
22. 
    
23. def execution_data(res):
    
24.     bs4_res = BeautifulSoup(res, 'html.parser')
    
25.     text = bs4_res.select('#app > div.lite-page-wrap > div > div.main > div > article > div > div > div.weibo-text')[0].text
    
26.     created_at = '"created_at": "(.*?)"'
    
27.     time_ = re.findall(created_at, res,re.S)
    
28.     return text,time_
    
29. 
    
30. data = pd.read_excel('wangzhi.xlsx')
    
31. 
    
32. text_all = []
    
33. time_all = []
    
34. for i in range(data.shape[0]-1):
    
35.     try:
    
36.         url = data['网址'].iloc[i]
    
37.         res = crawer(url)
    
38.         text = execution_data(res)[0]
    
39.         time_ = execution_data(res)[1]
    
40.         text_all.append(text)
    
41.         time_all.append(time_)
    
42.     except:
    
43.         print('第'+ str(i+1) + '个网页出现问题')
    
44.         print(url)
    
45.         text = []
    
46.         time_ = []
    
47.         text_all.append(text)
    
48.         time_all.append(time_)
    
49. 
    
50. context = pd.DataFrame({'文本':text_all,'时间':time_all})
    
51. context.to_excel('微博正文.xlsx')
    
52. print('爬取完成')
    
53. 
    
54. end_time = time.time()
    
55. total_time = end_time - start_time
    
56. print("所有任务结束，总耗时为：" + str(total_time))[postbg]bg8.png[/postbg]
    

复制代码