b站视频下载 爬虫

bloodbat007

单线程版本

1. #单线程版本
   
2. import requests,re,json
   
3. from pathlib import Path
   
4. from ffmpy3 import FFmpeg
   
5. from tqdm import tqdm
   
6. 
   
7. headers={"Cookie":"SESSDATA=这里填自己的",
   
8.     "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36 Edg/96.0.1054.41",
   
9.     "Referer":"https://www.bilibili.com/"}
   
10. 
    
11. def get_video_info(video_url:str):
    
12.     """[通过正则表达式定位文本 得到视频信息]
    
13. 
    
14.     Args:
    
15.         video_url (str): [视频链接]
    
16. 
    
17.     Returns:
    
18.         [type]: [包含视频信息的字典]
    
19.     """
    
20.     video_html=requests.get(video_url,headers=headers)
    
21. 
    
22.     #<span class="cur-page">(1/12)</span>
    
23.     pages=re.findall(r'<span class="cur-page">\((.*?)\)</span>', video_html.text)  #分页视频总数
    
24. 
    
25.     total_page=1
    
26.     if len(pages) !=0:
    
27.         cur_page=str(pages[0]).split('/')[0]
    
28.         total_page=int(str(pages[0]).split('/')[1])
    
29.         print(f'当前为多p视频总页数为{total_page},当前页数为{cur_page}')
    
30.     else:
    
31.         print('当前为单p视频')
    
32. 
    
33.     video_info={}
    
34. 
    
35.     for p in range(1,total_page+1):
    
36.         video_url=re.sub('p=.*', f'p={p}', video_url)
    
37.         video_html=requests.get(video_url,headers=headers)
    
38. 
    
39.         video_down_re=re.findall(r'<script>window\.__playinfo__=(.*?)</script>', video_html.text) #下载信息
    
40.         video_down_json=json.loads(video_down_re[0])
    
41. 
    
42.         video_base_re=re.findall(r'<script>window.__INITIAL_STATE__=(.*?);\(function\(\)', video_html.text) #基本信息
    
43.         video_base_json=json.loads(video_base_re[0])
    
44. 
    
45.         video_title=video_base_json['videoData']['pages'][p-1]['part']
    
46.         video_info[video_title]=dict(video_download_url=video_down_json['data']['dash']['video'][0]['baseUrl'],audio_download_url=video_down_json['data']['dash']['audio'][0]['baseUrl'])
    
47. 
    
48.     return video_info
    
49. 
    
50. def video_download(video_title:str,video_download_url:str,audio_download_url:str):
    
51.     """[下载并合拼视频和音频]
    
52. 
    
53.     Args:
    
54.         video_title:str 视频标题
    
55.         video_download_url (str): [视频下载链接]
    
56.         audio_download_url (str): [音频下载链接]
    
57.     """
    
58.     res=requests.get(url=video_download_url,headers=headers,stream=True)
    
59. 
    
60.     chunk_size=1024 #每次下载大小  byte
    
61.     content_size=int(res.headers['Content-Length']) # 总大小 byte
    
62.    
    
63.     # pbar=tqdm(total=content_size,desc=f'{video_title}.mp4',ncols=100,unit='byte',unit_scale=True)  #进度条
    
64.     # with open(f'{video_title}.mp4','wb') as stream:
    
65.     #     for data in res.iter_content(chunk_size=chunk_size):  #分块读取数据
    
66.     #         stream.write(data)
    
67.     #         pbar.update(len(data))
    
68.     # pbar.close()
    
69. 
    
70.     res=requests.get(url=audio_download_url,headers=headers)
    
71.     content_size=int(res.headers['Content-Length']) # 总大小 byte
    
72. 
    
73.     pbar=tqdm(total=content_size,desc=f'{video_title}.mp3',ncols=100,unit='byte',unit_scale=True)  #进度条
    
74.     with open(f'{video_title}.mp3','wb') as stream:
    
75.         for data in res.iter_content(chunk_size=chunk_size):  #分块读取数据
    
76.             stream.write(data)
    
77.             pbar.update(len(data))
    
78.     pbar.close()
    
79. 
    
80.     # #合并文件
    
81.     # ff = FFmpeg(inputs={f'{video_title}.mp4':None,f'{video_title}.mp3':None},outputs={f'{video_title}.mkv':'-codec copy'})
    
82.     # ff.run()
    
83.     # #删除文件
    
84.     # Path(f'{video_title}.mp3').unlink()
    
85.     # Path(f'{video_title}.mp4').unlink()
    
86.    
    
87. if __name__=='__main__':
    
88.     url='https://www.bilibili.com/video/BV1f3411B7HK?p=1'
    
89.     video_info=get_video_info(video_url=url)
    
90. 
    
91.     import time
    
92.     start_time=time.time()
    
93.     video_titles=video_info.keys()
    
94.     for title in video_titles:
    
95.         video_download(title,video_info[title]['video_download_url'],video_info[title]['audio_download_url'])
    
96.    
    
97.     print(f'time={time.time()-start_time}')
    
98.    

复制代码

异步协程版本

1. # 单线程异步协程版本
   
2. import asyncio
   
3. import re
   
4. import aiohttp
   
5. import aiofiles
   
6. import json
   
7. import requests
   
8. from tqdm.asyncio import tqdm
   
9. from pathlib import path
   
10. from ffmpy3 import FFmpeg
    
11. 
    
12. headers = {"Cookie": "SESSDATA=这里填自己的",
    
13.            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36 Edg/96.0.1054.41",
    
14.            "Referer": "https://www.bilibili.com/"}
    
15. 
    
16. 
    
17. async def get_video_pages(video_url: str) -> int:
    
18.     '''
    
19.     得到多p视频 总p数
    
20.     :param video_url:当前视频链接
    
21.     :return:总p数
    
22.     '''
    
23.     video_html = requests.get(url=video_url, headers=headers).text
    
24.     # <span class="cur-page">(1/12)</span>
    
25.     pages = re.findall(r'<span class="cur-page">\((.*?)\)</span>', video_html)  # 分页视频总数
    
26. 
    
27.     total_page = 1
    
28.     if len(pages) != 0:
    
29.         cur_page = str(pages[0]).split('/')[0]
    
30.         total_page = int(str(pages[0]).split('/')[1])
    
31.         print(f'当前为多p视频总页数为{total_page},当前页数为{cur_page}')
    
32.     else:
    
33.         print('当前为单p视频')
    
34. 
    
35.     return total_page
    
36. 
    
37. 
    
38. async def video_download(video_url: str, page: int):
    
39.     '''
    
40.     下载视频,音频,合并视频使用FFmpeg
    
41.     :param video_url:当前链接
    
42.     :param page: 当前p数
    
43.     :return: 无
    
44.     '''
    
45.     async with aiohttp.ClientSession() as session:
    
46.         async with session.get(url=video_url, headers=headers) as res:
    
47.             video_html = await res.text()
    
48. 
    
49.             video_down_re = re.findall(r'<script>window\.__playinfo__=(.*?)</script>', video_html)  # 下载信息
    
50.             video_down_json = json.loads(video_down_re[0])
    
51. 
    
52.             video_base_re = re.findall(r'<script>window.__INITIAL_STATE__=(.*?);\(function\(\)', video_html)  # 基本信息
    
53.             video_base_json = json.loads(video_base_re[0])
    
54. 
    
55.             video_title = video_base_json['videoData']['pages'][page - 1]['part']  # 标题
    
56.             video_download_url = video_down_json['data']['dash']['video'][0]['baseUrl']  # 视频链接
    
57.             audio_download_url = video_down_json['data']['dash']['audio'][0]['baseUrl']  # 音频链接
    
58. 
    
59.             # async with session.get(url=video_download_url, headers=headers) as res_video:
    
60.             #     video_byte=await res_video.read()
    
61.             async with session.get(url=audio_download_url, headers=headers) as res_audio:
    
62.                 audio_byte = await res_audio.read()
    
63.             # async with aiofiles.open(f'{video_title}.mp4','wb') as stream:
    
64.             #     await stream.write(video_byte)
    
65.             async with aiofiles.open(f'{video_title}.mp3', 'wb') as stream:
    
66.                 await stream.write(audio_byte)
    
67. 
    
68.             # # 合并文件
    
69.             # ff = FFmpeg(inputs={f'{video_title}.mp4': None, f'{video_title}.mp3': None},outputs={f'{video_title}.mkv': '-codec copy'})
    
70.             # ff.run()
    
71.             # # 删除文件
    
72.             # Path(f'{video_title}.mp3').unlink()
    
73.             # Path(f'{video_title}.mp4').unlink()
    
74. 
    
75. 
    
76. async def main():
    
77.     url = 'https://www.bilibili.com/video/BV1f3411B7HK?p=1'
    
78.     task_one = asyncio.create_task(get_video_pages(video_url=url))
    
79.     toatl_pages = await task_one
    
80. 
    
81.     tasks=[]
    
82.     for p in range(1, toatl_pages + 1):
    
83.         video_url = re.sub('p=.*', f'p={p}', url)
    
84.         tasks.append(video_download(video_url,p))
    
85. 
    
86.     import time
    
87.     start = time.time()
    
88.     with tqdm(asyncio.as_completed(tasks),total=len(tasks),unit='byte',unit_scale=True) as pbar:
    
89.         for j in pbar:
    
90.             await j
    
91.     print(f'time={time.time() - start}')
    
92. 
    
93. if __name__ == '__main__':
    
94.     loop=asyncio.new_event_loop()
    
95.     asyncio.set_event_loop(loop)
    
96.     loop.run_until_complete(main())
    

复制代码

槐月余八

厉害，可以请楼主帮忙看看这个问题吗爬虫代码有问题请各位大佬指点 [url]https://fishc.com.cn/thread-207084-1-1.html (出处: 鱼C论坛)[/url]