煎蛋网 爬虫练习更新

XTWLP

1. import urllib.request
   
2. import os
   
3. 
   
4. def url_open(url):
   
5.    
   
6.     req=urllib.request.Request(url)
   
7.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363')
   
8.     response = urllib.request.urlopen(req)
   
9.     html = response.read()
   
10.    
    
11.     return html
    
12. 
    
13. def get_page(url):
    
14.     html = url_open(url).decode('utf-8')
    
15.    
    
16.     a = html.find('Older Comments')+39
    
17.     b = html.find('"',a)
    
18.     print(html[a:b])
    
19.     return html[a:b]
    
20. 
    
21. def find_imgs(url):
    
22.    
    
23.     html = url_open(url).decode('utf-8')
    
24.     img_addrs = []
    
25.     a = html.find('img src=')
    
26.    
    
27.     while a != -1:
    
28.         b = html.find('.jpg',a,a+255)
    
29.         if b != -1:
    
30.             img_addrs.append("http:"+html[a+9:b+4])
    
31.         else:
    
32.             b = a+9
    
33.         a = html.find("img src=",b)
    
34.         
    
35.     return img_addrs
    
36. 
    
37. def save_imgs(folder, img_addrs):
    
38.     for each in img_addrs:
    
39.         print(each)
    
40.         filename = each.split('/')[-1]    #取最后一个部分
    
41.         with open(filename,'wb') as f:
    
42.             img=url_open(each)
    
43.             f.write(img)
    
44. 
    
45. def download_mm(folder='2', pages=10):
    
46.     os.mkdir(folder)
    
47.     os.chdir(folder)
    
48. 
    
49.     url = "/MjAyMDA4MDgtMTMx#comments"
    
50.    
    
51.     for i in range(pages):
    
52.         page_url = 'http://jandan.net/ooxx'+url
    
53.         print(page_url)
    
54.         img_addrs = find_imgs(page_url)
    
55.         save_imgs(folder, img_addrs)
    
56.         url = get_page(page_url)
    
57.         print(url)
    
58.         
    
59. download_mm()
    

复制代码

bai123tt

不错，进来学习一下