使用多线程1分钟下载腾讯视频20集35万条弹幕并存在数据库中

孟大大

1. import requests,re,time,random
   
2. import pymysql
   
3. from fake_useragent import UserAgent
   
4. from lxml import etree
   
5. import concurrent.futures
   
6. 
   
7. def get_danmu(params):#获取弹幕
   
8.     url=params['url']#获取弹幕url
   
9.     name = params['name']#获取数据库表明
   
10.     page=name.split('danmu')[1]#获取集数
    
11.     danmus=[]
    
12.     #建立代理ip池
    
13.     iplist = ['112.195.202.102:4278']#代理ip库,从网上选几个可用的
    
14.     ip = random.choice(iplist)
    
15.     proxies = {"http": ip, "https": ip}
    
16.     headers = {'User-Agent': UserAgent().random}#导入随机文件头库，用于反爬虫
    
17.     try:
    
18.         res=requests.get(url=url,headers=headers,proxies=proxies,timeout=10)#删除proxies=proxies使用本机ip下载
    
19.     except:
    
20.         j=0
    
21.         while j <10:#如果连接失败，重新进行连接，最多连接10次
    
22.             ip = random.choice(iplist)
    
23.             proxies = {"http": ip, "https": ip}
    
24.             try:
    
25.                 res = requests.get(url=url, headers=headers, proxies=proxies, timeout=10)
    
26.             except:
    
27.                 pass
    
28.             print('正在进行第{}次重新连接--------------------------------------------'.format(j))
    
29.             j += 1
    
30.             time.sleep(2)
    
31.     print('正在下载第{}集弹幕，本页弹幕共{}条'.format(page,eval(res.text)['count']))
    
32.     if eval(res.text)['count'] != 0 and res.status_code==200:#如果网页请求成功及弹幕数量大于0
    
33.         res = eval(res.text)#将网页相应内容变成字典
    
34.         for each in res['comments']:
    
35.             author = ''.join(re.findall(u"[\u4e00-\u9fa5]+", each['opername'].replace(' ', '')))#作者名字只提取中文
    
36.             content = ''.join(re.findall(u"[\u4e00-\u9fa5]+", each['content'].replace('\xa0', '')))#评论内容只提取中文
    
37.             danmus.append({
    
38.                 '作者': '未知作者' if len(author) == 0 else author,#如果为空值则默认为未知作者
    
39.                 '弹幕内容': '未知弹幕' if len(content) == 0 else content,#如果为空值则默认为未知弹幕
    
40.                 '点赞数': each['upcount'],
    
41.                 '弹幕发送时间': each['timepoint'],
    
42.                 '作者等级': each['uservip_degree'],
    
43.                 '弹幕id': each['commentid']})
    
44.     insert([tuple(each.values()) for each in danmus], name)#将提取的弹幕写入数据库，此处必须要以数组传入
    
45. 
    
46. 
    
47. 
    
48. def get_after_param():#获得后缀id
    
49.     url = 'https://v.qq.com/x/cover/o8mbrpo92gni5uc/i0036rltvk1.html'#输入任意电视剧某一集的网站
    
50.     headers = {'User-Agent': UserAgent().random}
    
51.     res = requests.get(url = url,headers=headers)
    
52.     html = etree.HTML(res.content)
    
53.     afterid = html.xpath("//div[@class='mod_episode']/span[@_stat='videolist:click' and not(i)]/@id")#不包含预告片及vip集数
    
54.     jishu = [each.replace(' ','').replace('\n','') for each in html.xpath("//div[@class='mod_episode']/span[@_stat='videolist:click' and not(i)]/a/text()")]
    
55.     after_param=dict(zip(jishu,afterid))#压缩成字典，方便数据传递
    
56.     return after_param
    
57. 
    
58. def get_before_param():#获得前缀id
    
59.     after_param=get_after_param()
    
60.     url='https://access.video.qq.com/danmu_manage/regist?vappid=97767206&vsecret=c0bdcbae120669fff425d0ef853674614aa659c605a613a4&raw=1'
    
61.     headers = {'User-Agent': UserAgent().random}
    
62.     for each in after_param:
    
63.         data = {"wRegistType": 2, "vecIdList": [after_param[each]], "wSpeSource": 0, "bIsGetUserCfg": 1,
    
64.                 "mapExtData": {after_param[each]: {"strCid": "o8mbrpo92gni5uc", "strLid": ""}}}
    
65.         res = requests.post(url=url, json=data, headers=headers).json()
    
66.         after_param[each]=res['data']['stMap'][after_param[each]]['strDanMuKey'].split('targetid=')[1]
    
67.     return after_param
    
68. 
    
69. def create_table(name):#创建表
    
70.     db = pymysql.connect(host='localhost', user='root', password='mxy', port=3306, db='python',charset='UTF8')
    
71.     cursor = db.cursor()
    
72.     cursor.execute("DROP TABLE IF EXISTS {}".format(name))
    
73.     sql = """CREATE TABLE {}
    
74.     (
    
75.     作者 varchar(255),
    
76.     弹幕内容 varchar(255),
    
77.     点赞数 varchar(255),
    
78.     弹幕发送时间 varchar(255),
    
79.     作者等级 varchar(255),
    
80.     弹幕id varchar(255)
    
81.     )""".format(name)
    
82.     cursor.execute(sql)
    
83.     db.close()
    
84. 
    
85. def insert(value,name):#插入数据
    
86.     db = pymysql.connect(host='localhost', user='root', password='mxy', port=3306, db='python')
    
87.     cursor = db.cursor()
    
88.     sql = "INSERT INTO {}(作者, 弹幕内容,点赞数, 弹幕发送时间,作者等级,弹幕id) values(%s, %s, %s,%s,%s,%s)".format(name)
    
89.     try:
    
90.         cursor.executemany(sql,value)
    
91.         db.commit()
    
92.     except:
    
93.         db.rollback()
    
94.         print(value)
    
95.     db.close()
    
96. 
    
97. def thread(after_param,name):
    
98.     urls = ['https://mfm.video.qq.com/danmu?target_id={}&timestamp={}'.format(after_param,i) for i in
    
99.             range(15, 3000, 30)]
    
100.     params=[]
     
101.     for url in urls:
     
102.         params.append({'url': url, 'name': name,'after_param':after_param})
     
103.     with concurrent.futures.ThreadPoolExecutor() as pool:
     
104.         htmls = pool.map(get_danmu,params)
     
105. 
     
106. 
     
107. def main():
     
108.     t1 = time.time()
     
109.     after_param = get_before_param()
     
110.     for i in range(1,len(after_param)+1):
     
111.         print('正在爬取第{}集-----------------------------------------------------------------'.format(i))
     
112.         t2=time.time()
     
113.         create_table('danmu'+str(i))
     
114.         thread(after_param[str(i)], 'danmu'+str(i))
     
115.         t3= time.time()
     
116.         print('-------------------------累计运行{}秒，本集弹幕爬取共用{}秒----------------------------------'.format(t3-t1,t3-t2))
     
117.     print('弹幕已经爬取完成')
     
118. 
     
119. main()

复制代码

昨非

yuedong

小伤口

哇塞i

hornwong

hornwong

very

very

very

crisfo

1073551873

3102515355

3553107168

.

1908496509