看了课堂上煎蛋的爬虫，用正则表达式略微简化
因为是几年前的视频，其实网页代码已经改变了不少了，最近看完视频，根据网页变化和前面一些课程的理解尝试了一下把视频中冗长的字符串搜索改成正则表达式来进行爬取，代码如下：import urllib.request
import os
import re
from bs4 import BeautifulSoup
def url_open(url):#模拟浏览器登录
req = urllib.request.Request(url)
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36')
response = urllib.request.urlopen(url)
html = response.read()
return html
def find_imgs(url):#获取图片地址，返回图片列表
soup = BeautifulSoup(url_open(url),'html.parser')
img_addrs = []
for each in soup.find_all(src=re.compile('.jpg')):
img_addrs.append('http:' + each['src'])
return img_addrs
def save_imgs(folder,img_addrs):#将每个图片地址打开保存
for each in img_addrs:
filename = each.split('/')[-1]
with open(filename,'wb') as f:
img = url_open(each)
f.write(img)
def download(folder='ooxx',page=10):#主函数，打印十页内容到‘ooxx’文件夹
os.mkdir(folder)
os.chdir(folder)
url = 'http://jandan.net/ooxx/'
for i in range(page):
img_addrs = find_imgs(url)
save_imgs(folder,img_addrs)
#下一页
soup = BeautifulSoup(url_open(url),'html.parser')
url = 'http:' + soup.find(title='Older Comments')['href']
if __name__ == '__main__':
download()
总的来说简短了不少
页:
[1]