|
发表于 2021-3-30 17:12:02
|
显示全部楼层
本楼为最佳答案
 - # -*- encoding = utf-8 -*-
- #@Time : 2020/9/5 11:30
- #@Author : XC
- #@File : movie_rank.py
- #@Software : pycharm
- import requests
- from bs4 import BeautifulSoup
- from lxml import etree
- import xlwt
- import time
- import os
- #1,获取一个网页
- def gethtml(url):
- try:
- header = {'User-agent': "Mozilla/5.0 (Windows NT 6.3; Win64; x64) "
- "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36"
- }
- html = requests.get(url=url,headers=header)
- html.raise_for_status()
- html.encoding = html.apparent_encoding
- return html.text
- except:
- print("解析异常")
- #2,解析多个网页内容,提取要素---用 bs4 和 正则表达式 分析数据
- def analysis(baseurl):
- for i in range(page):
- url = baseurl + str(i * 25)
- html = gethtml(url) #函数嵌套
- bs = BeautifulSoup(html,"html.parser")
- #找到一个电影的完整的信息
- for item in bs.find_all('div',class_="item"):
- item = str(item) #可以直接用xpath解析出所有的item,然后在每个item中解析
- data = [] #一个电影的完整信息
- #构建xpath对象
- demo_item = etree.HTML(item)
- link = demo_item.xpath('//div[@class="hd"]/a/@href')
- data.append(link[0])
- pic = demo_item.xpath('//div[@class="pic"]/a/img/@src')
- data.append(pic[0])
- cname = demo_item.xpath('//div[@class="hd"]/a/span[1]/text()')
- data.append(cname[0])
- oname = demo_item.xpath('//div[@class="hd"]/a/span[2]/text()') #判断外文名为空
- if len(oname) != 0:
- data.append(oname[0])
- else:
- data.append('')
- inp1 = demo_item.xpath('//div[@class="bd"]/p[1]/text()[1]') #可能为空
- if len(inp1) != 0:
- inp1[0] = inp1[0].replace(' ','').replace(r'\n','')
- data.append(inp1[0])
- else:
- data.append('')
- inp2 = demo_item.xpath('//div[@class="bd"]/p[1]/text()[2]') #可能为空
- if len(inp2) != 0:
- inp2[0] = inp2[0].replace(' ', '').replace(r'\n', '')
- data.append(inp2[0])
- else:
- data.append('')
- score = demo_item.xpath('//div[contains(@class,"star")]/span[2]/text()')
- data.append(score[0])
- express = demo_item.xpath('//div[contains(@class,"star")]/span[last()]/text()')
- express[0] = express[0].replace('人评价','')
- data.append(express[0])
- last = demo_item.xpath('//div[@class="bd"]/p[2]/span/text()') #可能为空
- if len(last) != 0:
- last[0] = last[0].replace('。','')
- data.append(last[0])
- else:
- data.append('')
- yield data #一部电影的所有的信息
- #3,保存
- def save(path):
- wb = xlwt.Workbook(encoding='utf-8')
- ws = wb.add_sheet('sheet',cell_overwrite_ok=True)
- num = 1
- col = ['链接','图片','中文名','外文名','概况','类型','评分','评价数','介绍']
- for i in range(9):
- ws.write(0,i,col[i]) #记录表头
- for each in analysis(baseurl=baseurl): #实现生成器传值翻页处理,each表示一个电影的全部信息
- print('开始爬取...')
- for element in each: #element表示一个电影的各个元素
- print(num,each.index(element),element)
- ws.write(num,each.index(element),element)
- if each.index(element) == 1:
- # pic_html = gethtml(element)
- # with open(element[2],'wb') as f:
- # f.write(pic_html)
- # f.close()
- # print('save success! the %d picture' % num )
- putpic(each[2],element)
- print('第 %d 张图片保存成功!' % num )
- num += 1
- wb.save(path)
- print("保存成功 %d 个电影信息" % (num-1) )
- """输出图片"""
- def putpic(pic_name, pic_url, root="G://pic//"): # root为保存文件的位置
- file_name = pic_name
- pic_path = root + file_name + '.jpg'
- try:
- if not os.path.exists(root):
- os.mkdir(root)
- else:
- if not os.path.exists(pic_path):
- response = requests.get(pic_url)
- with open(pic_path, 'wb') as f:
- f.write(response.content)
- time.sleep(0.2)
- f.close()
- except:
- pass
- if __name__ == '__main__':
- #init
- baseurl = 'https://movie.douban.com/top250?start='
- page = 1 #只有十页豆瓣网页
- save(path='movie_douban_2.xls')
- #实现了从一个网页中打开和爬取多个网页的信息,从一个网页中爬取到主要的列表信息,以及每一信息对应的图片。
复制代码
运行结果如下图:
|
|