爬虫问题求助

代码小白liu

import requests
from lxml import etree

url = "https://www.pearvideo.com/category_4"
headers= {
    "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Mobile Safari/537.36"
}

response = requests.get(url=url, headers=headers)
html_text = response.text
tree = etree.HTML(html_text)
li_list = tree.xpath('//div[@id="listvideoList"]/ul/li')

for li in li_list:
    detal_url = 'https://www.pearvideo.com/' + li.xpath('./div/a/@href')[0]
    name = li.xpath('./div/a/div[2]/text()')[0] + '.mp4'
    detal_response = requests.get(url=detal_url, headers=headers).text
    

以上是爬取梨视频的代码，视频网页的地址是找到了，可是要找下载视频的地址没找到，网页源代码也没有请教下，没有的话这时候该如何进行思考分析，该怎么找地址，给点思路代码指引下，谢谢了！

suchocolate

import requests
from lxml import etree
import random
import json


def main():
    result = []  # 存储视频链接
    url = 'https://www.pearvideo.com/category_4'  # 总入口
    base_url = 'https://www.pearvideo.com/'   # 用于合成url和referer
    headers = {'user-agent': 'firefox'}
    r = requests.get(url, headers=headers)
    html = etree.HTML(r.text)
    v_pages = html.xpath('//a[contains(@class,"vervideo-lilink")]/@href')   # 获得各视频的页面的url
    for v_page in v_pages:  # 遍历各视频页面的url，从url中获取视频编号
        # example: v_page like video_1743082
        num = v_page.split('_')[1]   # 视频编号，用于合成视频信息url，视频信息url浏览器f12-网络就能看到，如下：
        # example: https://www.pearvideo.com/videoStatus.jsp?contId=1743082&mrd=0.554376234182324
        url = f'{base_url}videoStatus.jsp?contId={num}&mrd={random.random()}'
        # example: Referer: https://www.pearvideo.com/video_1743082
        headers['Referer'] = f'{base_url}{v_page}'  # 想要获得视频信息，需要在头中添加referer
        r = requests.get(url, headers=headers)
        data = json.loads(r.text)  # 视频信息是以json形式反馈的
        # print(data['videoInfo']['videos']['srcUrl'])
        result.append(data['videoInfo']['videos']['srcUrl'])   # 把视频连接放到结果列表中
    print(result)


if __name__ == '__main__':
    main()

代码小白liu

suchocolate 发表于 2021-10-7 20:37

谢谢大佬指导，json 这块我还没学明白，我先去学习下。谢谢