爬虫那一节，跟着小甲鱼老师后面敲的，怎么不对

happy_study

1. 
   
2. 
   
3. import urllib.request
   
4. import os
   
5. 
   
6. def url_open(url):
   
7.     req = urllib.request.Request(url)
   
8.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36')
   
9.     response = urllib.request.urlopen(url)
   
10.     html = response.read()
    
11.    
    
12. 
    
13.     return html
    
14. 
    
15. 
    
16. def get_page(url):
    
17.     html = url_open(url).decode('utf-8')
    
18. 
    
19.     a = html.find('current-comment-page')+23
    
20.     b = html.find(']',a)
    
21. 
    
22.     return html[a:b]
    
23. 
    
24.    
    
25. def find_imgs(url):
    
26.     html = url_open(url).decode('utf-8')
    
27.     img_addrs = []
    
28. 
    
29.     a = html.find('img src=')
    
30.    
    
31.     while a != -1:
    
32.         b = html.find('.jpg',a,a+255)
    
33.         if b != -1:
    
34.             img_addrs.append(html[a+9:b+4])
    
35.         else:
    
36.             b = a + 9
    
37. 
    
38. 
    
39.         a = html.find('img src=',b)
    
40. 
    
41.     return img_addrs
    
42.         
    
43. 
    
44. def save_imgs(folder,img_addrs):
    
45.     for each in img_addrs:
    
46.         filename = each.split('/')[-1]
    
47.         with open(filename,wb) as f:
    
48.             img = url_open(each)
    
49.             f.write(img)
    
50. 
    
51. 
    
52. 
    
53. def download_mm(folder='OOXX',pages=10):
    
54.     os.mkdir(folder)
    
55.     os.chdir(folder)
    
56. 
    
57.     url = "http://jandan.net/ooxx/"
    
58.     page_num = int(get_page(url))
    
59. 
    
60.     for i in range(pages):
    
61.         page_num -= i
    
62.         page_url = url + 'page-' + str(page_num) + '#comments'
    
63.         img_addrs = find_imgs(page_url)
    
64.         save_imgs(folder,img_addrs)
    
65. 
    
66. if __name__=='__main__':
    
67.     download_mm()
    

复制代码

wongyusing

网站改版了，这份代码爬不了

happy_study

wongyusing 发表于 2018-12-29 14:00
网站改版了，这份代码爬不了

居然遇到这操作