|
马上注册,结交更多好友,享用更多功能^_^
您需要 登录 才可以下载或查看,没有账号?立即注册
x
#encoding: utf-8
import requests
from lxml import etree
BASE_DOMAIN = 'http://www.dytt8.net'
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134'}
def get_detail_urls(url):
response = requests.get(url,headers=HEADERS )
text = response.content.decode('gbk','ignore')
html = etree.HTML(text)
detail_urls=html.xpath("//table[@class='tbspan']//a/@href")
detail_urls = map(lambda url:BASE_DOMAIN + url,detail_urls )
return detail_urls
def parse_detail_page(url):
movie = {}
response = requests.get(url,headers=HEADERS )
text = response.content.decode('gbk','ignore')
html = etree.HTML(text)
title = html.xpath("//div[@class='title_all']/font[@color='#07519a']/text()")
movie['title'] = title
zoomE = html.xpath("//div[@id='Zoom']")[0]
imgs =zoomE.xpath(".//img/@src")
cover = imgs[0] #if imgs else None
screenshot = imgs[1] #if imgs else None
movie['cover'] = cover
movie['screenshot'] = screenshot
def parse_info(info,rule):
return info.replace(rule,'').strip()
infos =zoomE.xpath ("//text()")
for index,info in enumerate (infos):
if info.startswith ('◎译 名'):
info = parse_info(info,'◎译 名')
movie['name'] = info
if info.startswith('◎年 代'):
info = parse_info(info,'◎年 代')
movie['year'] = info
elif info.startswith('◎产 地'):
info = parse_info(info,'◎产 地')
movie['country'] = info
elif info.startswith ('◎类 别'):
info = parse_info(info,'◎类 别')
movie['category'] = info
elif info.startswith ('◎豆瓣评分'):
info = parse_info(info,'◎豆瓣评分')
movie['douban_rating'] = info
elif info.startswith ("◎片 长"):
info = parse_info(info,'◎片 长')
movie['duratinon'] = info
elif info.startswith ('◎导 演'):
info = parse_info(info,'◎导 演')
movie['direcotr'] = info
elif info.startswith ('◎主 演'):
info = parse_info(info,'◎主 演')
actors = [info]
for x in range(index+1,len(infos) ):
actor = infos[x]
if actor.startswith('◎'):
break
actors.append(actor)
movie['actor'] = actors
elif info.startswith('◎简 介'):
info = parse_info(info,'◎简 介')
movie['profile'] = info
download_url = html.xpath("//td[@bgcolor='#fdfddf']/a/@herf")
print(download_url )
#movie['download_url'] = download_url
#return movie
def spider():
base_url = 'http://www.dytt8.net/html/gndy/dyzz/list_23_{}.html'
movies = []
for x in range(1,8):
url = base_url.format(x)
detail_urls = get_detail_urls(url)
for detail_url in detail_urls :
#用来遍历一夜中的所有电影详情的url
movie = parse_detail_page(detail_url )
movies.append(movie)
print(movie)
#print(movies)
if __name__ == '__main__':
#parse_detail_page()
spider() |
|