 # -*- encoding = utf-8 -*-
#@Time : 2020/9/5 11:30
#@Author : XC
#@File : movie_rank.py
#@Software : pycharm
import requests
from bs4 import BeautifulSoup
from lxml import etree
import xlwt
import time
import os
#1,获取一个网页
def gethtml(url):
try:
header = {'User-agent': "Mozilla/5.0 (Windows NT 6.3; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36"
}
html = requests.get(url=url,headers=header)
html.raise_for_status()
html.encoding = html.apparent_encoding
return html.text
except:
print("解析异常")
#2，解析多个网页内容，提取要素---用 bs4 和 正则表达式 分析数据
def analysis(baseurl):
for i in range(page):
url = baseurl + str(i * 25)
html = gethtml(url) #函数嵌套
bs = BeautifulSoup(html,"html.parser")
#找到一个电影的完整的信息
for item in bs.find_all('div',class_="item"):
item = str(item) #可以直接用xpath解析出所有的item，然后在每个item中解析
data = [] #一个电影的完整信息
#构建xpath对象
demo_item = etree.HTML(item)
link = demo_item.xpath('//div[@class="hd"]/a/@href')
data.append(link[0])
pic = demo_item.xpath('//div[@class="pic"]/a/img/@src')
data.append(pic[0])
cname = demo_item.xpath('//div[@class="hd"]/a/span[1]/text()')
data.append(cname[0])
oname = demo_item.xpath('//div[@class="hd"]/a/span[2]/text()') #判断外文名为空
if len(oname) != 0:
data.append(oname[0])
else:
data.append('')
inp1 = demo_item.xpath('//div[@class="bd"]/p[1]/text()[1]') #可能为空
if len(inp1) != 0:
inp1[0] = inp1[0].replace(' ','').replace(r'\n','')
data.append(inp1[0])
else:
data.append('')
inp2 = demo_item.xpath('//div[@class="bd"]/p[1]/text()[2]') #可能为空
if len(inp2) != 0:
inp2[0] = inp2[0].replace(' ', '').replace(r'\n', '')
data.append(inp2[0])
else:
data.append('')
score = demo_item.xpath('//div[contains(@class,"star")]/span[2]/text()')
data.append(score[0])
express = demo_item.xpath('//div[contains(@class,"star")]/span[last()]/text()')
express[0] = express[0].replace('人评价','')
data.append(express[0])
last = demo_item.xpath('//div[@class="bd"]/p[2]/span/text()') #可能为空
if len(last) != 0:
last[0] = last[0].replace('。','')
data.append(last[0])
else:
data.append('')
yield data #一部电影的所有的信息
#3，保存
def save(path):
wb = xlwt.Workbook(encoding='utf-8')
ws = wb.add_sheet('sheet',cell_overwrite_ok=True)
num = 1
col = ['链接','图片','中文名','外文名','概况','类型','评分','评价数','介绍']
for i in range(9):
ws.write(0,i,col[i]) #记录表头
for each in analysis(baseurl=baseurl): #实现生成器传值翻页处理,each表示一个电影的全部信息
print('开始爬取...')
for element in each: #element表示一个电影的各个元素
print(num,each.index(element),element)
ws.write(num,each.index(element),element)
if each.index(element) == 1:
# pic_html = gethtml(element)
# with open(element[2],'wb') as f:
# f.write(pic_html)
# f.close()
# print('save success! the %d picture' % num )
putpic(each[2],element)
print('第 %d 张图片保存成功！' % num )
num += 1
wb.save(path)
print("保存成功 %d 个电影信息" % (num-1) )
"""输出图片"""
def putpic(pic_name, pic_url, root="G://pic//"): # root为保存文件的位置
file_name = pic_name
pic_path = root + file_name + '.jpg'
try:
if not os.path.exists(root):
os.mkdir(root)
else:
if not os.path.exists(pic_path):
response = requests.get(pic_url)
with open(pic_path, 'wb') as f:
f.write(response.content)
time.sleep(0.2)
f.close()
except:
pass
if __name__ == '__main__':
#init
baseurl = 'https://movie.douban.com/top250?start='
page = 1 #只有十页豆瓣网页
save(path='movie_douban_2.xls')
#实现了从一个网页中打开和爬取多个网页的信息，从一个网页中爬取到主要的列表信息，以及每一信息对应的图片。
运行结果如下图：
| 
( 粤ICP备18085999号-1 | 粤公网安备 44051102000585号)
狗仔卡
发表于 2021-3-30 14:26:01
置顶卡
千斤顶
显身卡
楼主
发表于 2021-3-30 17:12:02