各位大大看下这爬虫
import requestsimport re
import os
from bs4 import BeautifulSoup
import time
def header(url):# 头文件，反反爬
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
'Referer': url}
return headers
def get_html_text(url):# 通用爬虫代码
try:
r = requests.get(url, headers=header(url))
r.raise_for_status()
r.encoding = 'utf-8'
return r.text
except:
print("爬取失败！")
def get_all_page_list(num):# 获取指定爬取数量的页面地址
all_page_list = []
for i in range(num):
all_page_list.append(f"https://www.mzitu.com/page/{i+1}/")# 修改后面的数，将从这页开始爬，比方说上次爬到第6页，这次从第7页开始，就把i+1改成i+7
return all_page_list
def get_page_group_list(url):# 获取单页每组图片地址
html = get_html_text(url)
p = r'<li><a href="(https://www.mzitu.com/\d*)'
page_group_list = re.findall(p, html)
return page_group_list
def get_every_group_pic_list(url):# 获取每组所有图片的地址
html = get_html_text(url)
soup = BeautifulSoup(html, 'lxml')
page_num = int(soup.find('div', 'pagenavi').find_all('span')[-2].string)# 获取每组图片的页数
every_group_pic_list = []
for i in range(page_num):
every_group_pic_list.append(f"{url}/{i+1}")
return every_group_pic_list
def get_save_pic(url): # 获取和保存图片文件
html = get_html_text(url)
soup = BeautifulSoup(html, 'lxml')
pic_src = soup.find('div', 'main-image').find('img')['src']
pic_name = pic_src.split('/')[-1]
root = "D://妹子图//"
path = root + soup.find('div', 'main-image').find('img')['alt'] + '//'
try:
os.mkdir(path)
os.chdir(path)
except FileExistsError:
pass
with open(pic_name, 'wb') as f:
f.write(requests.get(pic_src, headers=header(pic_src)).content)
def main():
num = int(input("请输入爬取的页数(共200多页，每页有24组图片)："))
all_page_list = get_all_page_list(num)# 获取需要爬取的总页数地址列表
print(all_page_list)
print("====================================================================")
for url1 in all_page_list:
page_group_list = get_page_group_list(url1)# 获取单页每组图片地址列表
print(page_group_list)
print("====================================================================")
for url2 in page_group_list:
every_group_pic_list = get_every_group_pic_list(url2)# 获取每组所有图片地址列表
for url3 in every_group_pic_list:
print(url3)
print("====================================================================")
get_save_pic(url3)# 获取并保存每组图片
time.sleep(1) # 一定要保留，如果太快，最多下两个图集就停了
try:
os.mkdir("D://妹子图")# 下载的图片存放在D盘的妹子图文件夹里
except:
pass
main()
各位大大看下这段爬虫哪里有问题，昨天运行的时候提示无法链接主机
页:
[1]