爬虫问题求助
import requestsfrom lxml import etree
url = "https://www.pearvideo.com/category_4"
headers= {
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Mobile Safari/537.36"
}
response = requests.get(url=url, headers=headers)
html_text = response.text
tree = etree.HTML(html_text)
li_list = tree.xpath('//div[@id="listvideoList"]/ul/li')
for li in li_list:
detal_url = 'https://www.pearvideo.com/' + li.xpath('./div/a/@href')
name = li.xpath('./div/a/div/text()') + '.mp4'
detal_response = requests.get(url=detal_url, headers=headers).text
以上是爬取梨视频的代码，视频网页的地址是找到了，可是要找下载视频的地址没找到，网页源代码也没有请教下，没有的话这时候该如何进行思考分析，该怎么找地址，给点思路代码指引下，谢谢了！
本帖最后由 suchocolate 于 2021-10-7 20:57 编辑
import requests
from lxml import etree
import random
import json
def main():
result = []# 存储视频链接
url = 'https://www.pearvideo.com/category_4'# 总入口
base_url = 'https://www.pearvideo.com/' # 用于合成url和referer
headers = {'user-agent': 'firefox'}
r = requests.get(url, headers=headers)
html = etree.HTML(r.text)
v_pages = html.xpath('//a/@href') # 获得各视频的页面的url
for v_page in v_pages:# 遍历各视频页面的url，从url中获取视频编号
# example: v_page like video_1743082
num = v_page.split('_') # 视频编号，用于合成视频信息url，视频信息url浏览器f12-网络就能看到，如下：
# example: https://www.pearvideo.com/videoStatus.jsp?contId=1743082&mrd=0.554376234182324
url = f'{base_url}videoStatus.jsp?contId={num}&mrd={random.random()}'
# example: Referer: https://www.pearvideo.com/video_1743082
headers['Referer'] = f'{base_url}{v_page}'# 想要获得视频信息，需要在头中添加referer
r = requests.get(url, headers=headers)
data = json.loads(r.text)# 视频信息是以json形式反馈的
# print(data['videoInfo']['videos']['srcUrl'])
result.append(data['videoInfo']['videos']['srcUrl']) # 把视频连接放到结果列表中
print(result)
if __name__ == '__main__':
main()
suchocolate 发表于 2021-10-7 20:37
谢谢大佬指导，json 这块我还没学明白，我先去学习下。谢谢
页:
[1]