多线程-爬取-度小视部分搞笑视频。

yfcz6095

1. import requests
   
2. import os
   
3. import re
   
4. from lxml import etree
   
5. from multiprocessing.dummy import Pool
   
6. 
   
7. 
   
8. os.chdir('D:\\Desktop\\Python练习小程序\\练习用文本文档\\视频')
   
9. headers = {
   
10.             'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.67'   
    
11.         }
    
12. 
    
13. url = 'http://quanmin.baidu.com/tab/funny'
    
14. res = requests.get(url,headers=headers)
    
15. html = res.text
    
16. tree = etree.HTML(html)
    
17. 
    
18. h_list = tree.xpath('//div[@class="card-video-list"]//h4')
    
19. #print(h_list)
    
20. urls = [] #存储所有视频播放地址
    
21. 
    
22. def 保留中文(string):
    
23.     pattern = re.compile(r'[^\u4e00-\u9fa5]')
    
24.     chinese = re.sub(pattern,'',string)
    
25.     return chinese
    
26. 
    
27. 
    
28. 
    
29. for h in h_list:
    
30.     sp_url = h.xpath('./a/@href')[0]   
    
31.     sp_mingzi = h.xpath('./a/text()')
    
32.     if len(sp_mingzi) != 0:
    
33.         sp_mingzi = 保留中文(sp_mingzi[0]) + '.mp4'
    
34.     else:
    
35.         sp_mingzi = 'kong.mp4'
    
36.     #print(sp_url,sp_mingzi)
    
37. 
    
38.       
    
39.     #对详情页的视频发起请求
    
40.     sp_text = requests.get(url=sp_url,headers=headers).text
    
41.    
    
42.     src = re.findall(r'<meta property="og:videosrc" content="(.*?)">',sp_text)[0]
    
43.     #print(src)
    
44.    
    
45.     dic = {
    
46.         'name':sp_mingzi,
    
47.         'url':src
    
48.     }
    
49.     urls.append(dic)
    
50. #print(urls)   
    
51.    
    
52. def get_video_data(dic):#保存视频函数
    
53.     url = dic['url']
    
54.     print(dic['name'],'正在下载......')
    
55.     data = requests.get(url=url,headers=headers).content
    
56.     with open(dic['name'],'wb') as fp:
    
57.         fp.write(data)
    
58.         print(dic['name'],'下载成功')
    
59. #使用线程池对视频播放地址src请求（很耗时的阻塞操作）
    
60. pool = Pool(5)
    
61. pool.map(get_video_data,urls)
    
62. 
    
63. pool.close()#关闭线程池
    
64. pool.join()#主线程等待子线程结束后再结束
    
65.    
    
66. 
    
67. 
    

复制代码

wuqramy

sofa

深谙流年