多线程-爬取-度小视部分搞笑视频。

yfcz6095

import requests
import os
import re
from lxml import etree
from multiprocessing.dummy import Pool


os.chdir('D:\\Desktop\\Python练习小程序\\练习用文本文档\\视频')
headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.67'    
        }

url = 'http://quanmin.baidu.com/tab/funny'
res = requests.get(url,headers=headers)
html = res.text
tree = etree.HTML(html)

h_list = tree.xpath('//div[@class="card-video-list"]//h4')
#print(h_list)
urls = [] #存储所有视频播放地址

def 保留中文(string):
    pattern = re.compile(r'[^\u4e00-\u9fa5]')
    chinese = re.sub(pattern,'',string)
    return chinese



for h in h_list:
    sp_url = h.xpath('./a/@href')[0]   
    sp_mingzi = h.xpath('./a/text()')
    if len(sp_mingzi) != 0:
        sp_mingzi = 保留中文(sp_mingzi[0]) + '.mp4'
    else:
        sp_mingzi = 'kong.mp4'
    #print(sp_url,sp_mingzi)

       
    #对详情页的视频发起请求
    sp_text = requests.get(url=sp_url,headers=headers).text
    
    src = re.findall(r'<meta property="og:videosrc" content="(.*?)">',sp_text)[0]
    #print(src)
    
    dic = {
        'name':sp_mingzi,
        'url':src
    }
    urls.append(dic)
#print(urls)    
   
def get_video_data(dic):#保存视频函数
    url = dic['url']
    print(dic['name'],'正在下载......')
    data = requests.get(url=url,headers=headers).content
    with open(dic['name'],'wb') as fp:
        fp.write(data)
        print(dic['name'],'下载成功')
#使用线程池对视频播放地址src请求（很耗时的阻塞操作）
pool = Pool(5)
pool.map(get_video_data,urls)

pool.close()#关闭线程池
pool.join()#主线程等待子线程结束后再结束
   

wuqramy

sofa

深谙流年