XTWLP 发表于 2020-8-8 20:29:46

煎蛋网 爬虫练习更新

import urllib.request
import os

def url_open(url):
   
    req=urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363')
    response = urllib.request.urlopen(req)
    html = response.read()
   
    return html

def get_page(url):
    html = url_open(url).decode('utf-8')
   
    a = html.find('Older Comments')+39
    b = html.find('"',a)
    print(html)
    return html

def find_imgs(url):
   
    html = url_open(url).decode('utf-8')
    img_addrs = []
    a = html.find('img src=')
   
    while a != -1:
      b = html.find('.jpg',a,a+255)
      if b != -1:
            img_addrs.append("http:"+html)
      else:
            b = a+9
      a = html.find("img src=",b)
      
    return img_addrs

def save_imgs(folder, img_addrs):
    for each in img_addrs:
      print(each)
      filename = each.split('/')[-1]    #取最后一个部分
      with open(filename,'wb') as f:
            img=url_open(each)
            f.write(img)

def download_mm(folder='2', pages=10):
    os.mkdir(folder)
    os.chdir(folder)

    url = "/MjAyMDA4MDgtMTMx#comments"
   
    for i in range(pages):
      page_url = 'http://jandan.net/ooxx'+url
      print(page_url)
      img_addrs = find_imgs(page_url)
      save_imgs(folder, img_addrs)
      url = get_page(page_url)
      print(url)
      
download_mm()

bai123tt 发表于 2020-8-13 21:36:46

不错，进来学习一下{:10_245:}
页: [1]
查看完整版本: 煎蛋网 爬虫练习更新