爬取百度贴吧图片
emm，本人也是刚学python一个月，也不是编程专业的，没什么编程基础，昨天试着自己爬取百度贴吧图片，没想到成功了，今天来分享给大家代码还有很多不足，大家可以提出修改意见，本人也用这个代码爬取到了想要的图片{:5_97:}
运行代码可以爬取想要爬取的贴吧，比如说想爬取女神贴吧，直接输入女神就行了
import re
import requests as rq
import os
import easygui as e
'''执行完后page_url就存放了这个贴吧页数的链接'''
def get_page(object_url):
page_url = {}
'''字典保存每一页和链接'''
'''进去关键字就是第一页'''
page_url = object_url
r1 = re.compile(r'<a.href="(//tieba.*?)".class=".pagination-item.".>(\d)</a>',re.S)
'''抓取页数链接'''
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}
response = rq.get(object_url,headers=headers).text
list1 = re.findall(r1,response)
for each in list1:
temp = ''.join(['https:', each])
page_url)] = temp
return page_url
'''传入哪一页，然后爬取这一页的贴子，并保存，这里只能传入页的url,temp是当前页数'''
def search_post(page_url,temp,object):
response = rq.get(page_url).text
r1 = re.compile(r'<a\srel="noreferrer".*?href="(/p/.*?)"', re.S)
'''抓取每个贴'''
each_result = re.findall(r1, response)
assert each_result != []
post_url = []
'''存放这一页所有贴的url'''
all_img_url = []
'''存放这一页所有满足条件的图片url'''
for each in each_result:
post_url.append(''.join(['https://tieba.baidu.com', each]))
'''传回每一页的贴的链接'''
r2 = re.compile(r'<img\sclass="BDE.*?src="(.*?)"', re.S)
'''抓取每个贴中的图片'''
for i in range(len(post_url)):
try:
response = rq.get(post_url, timeout=5).text
except:
pass
print('正在爬取第%d个贴.....' % i)
img_url = re.findall(r2, response)
'''这里抓取每个贴中的图片
如果该贴中图片过少，则不爬取'''
if len(img_url) <= object:
continue
else:
'''如果该贴图片足够，则统一放到一个列表中'''
for each_img in img_url:
all_img_url.append(each_img)
save_file(all_img_url,temp,object)
def save_file(all_img_url,page_num,object):
new_path = object + '/第%d页' % page_num
os.mkdir(new_path)
os.chdir(new_path)
count = 0
for each in all_img_url:
count += 1
with open(str(count) + '.jpg', 'wb') as f:
f.write(rq.get(each).content)
print('正在保存...')
os.chdir('..')
def main():
flag = 1
while flag:
object = init()
'''如果用户直接退出，则退出程序'''
if object == 0:
break
'''0是url，1是设置的爬取页数'''
page_url = get_page(object)
for each_page in range(len(page_url)):
temp = each_page
temp += 1
if temp > object:
choose = e.ccbox('是否继续爬取？',choices=['Yes', 'No'])
if choose != True:
break
try:
search_post(page_url,temp,object)
flag = 0
except:
e.msgbox('该贴吧不存在，请重新输入哦')
def init():
'''初始设置'''
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}
post_name = e.enterbox('请输入想爬取的贴吧名称...')
data = {
'ie': 'utf-8',
'kw': post_name,
'fr': 'search'
}
object_url = rq.get('https://tieba.baidu.com/f', params=data, headers=headers).url
filter_num = e.integerbox('请输入数字,比如输入2，则图片少于5的贴不爬取',default=2)
page_num = e.integerbox('请输入数字,比如输入2，则爬取2页',default=2)
if filter_num != None:
'''1.创建文件夹，保存图片'''
file_path = e.diropenbox('创建新的文件夹，选择图片存放路径路径')
file_name = e.enterbox('设置文件夹名字',default='%s图片'%post_name)
file_path = file_path + '/' + file_name
os.mkdir(file_path)
os.chdir(file_path)
return object_url,page_num,file_path,filter_num
else:
e.msgbox('程序已经退出')
return 0
main()
必须得赞
页:
[1]