[已解决]问题求助

小尤you

煎蛋网爬取图片不成功该怎么解决？
代码如下：

1. 
   
2. import urllib.request
   
3. import os
   
4. 
   
5. def url_open(url):
   
6.     data=None
   
7.     req = urllib.request.Request(url)
   
8.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 LBBROWSER')
   
9.     response = urllib.request.urlopen(req)
   
10.     html = response.read()
    
11. 
    
12.     return html
    
13. 
    
14. def get_page(url):
    
15.     html = url_open(url).decode('utf-8')
    
16. 
    
17.     a = html.find('current-comment-page') + 23
    
18.     b = html.find(']',a)
    
19. 
    
20.     return html[a:b]
    
21.    
    
22. 
    
23. def find_imgs(url):
    
24.     html = url_open(url).decode('utf-8')
    
25.     img_addrs = []
    
26. 
    
27.     a = html.find('img src=')
    
28. 
    
29.     while a != -1:
    
30.         b = html.find('.jpg',a,a+255)
    
31.         if b !=-1:
    
32.             img_addrs.append('http:' + html[a+9:b+4])
    
33.         else:
    
34.             b =a + 9
    
35. 
    
36.         a = html.find('img src=',b)
    
37. 
    
38.     return img_addrs
    
39. 
    
40.         
    
41. def save_imgs(folder,img_addrs):
    
42.     for each in img_addrs:
    
43.         filename = each.split('/')[-1]
    
44.         with open(filename,'wb') as f:
    
45.             img = url_open(each)
    
46.             f.write(img)
    
47.             
    
48. 
    
49. def download_mm(folder='OOXX',pages=10):
    
50.     os.mkdir(folder)
    
51.     os.chdir(folder)
    
52. 
    
53.     url = "http://jandan.net/ooxx/"
    
54.     page_num = int(get_page(url))
    
55. 
    
56.     for i in range(pages):
    
57.         page_num -= i
    
58.         page_url = url + 'page-' + str(page_num) + '#comments'
    
59.         img_addrs = find_imgs(page_url)
    
60.         save_imgs(folder,img_addrs)
    
61.         time.sleep(10)
    
62. 
    
63. if __name__ == '__main__':
    
64.     download_mm()
    

复制代码

报错如下
Traceback (most recent call last):
  File "E:\Python学习\小甲鱼课上\下载妹子图.py", line 63, in <module>
    download_mm()
  File "E:\Python学习\小甲鱼课上\下载妹子图.py", line 58, in download_mm
    img_addrs = find_imgs(page_url)
  File "E:\Python学习\小甲鱼课上\下载妹子图.py", line 23, in find_imgs
    html = url_open(url).decode('utf-8')
  File "E:\Python学习\小甲鱼课上\下载妹子图.py", line 8, in url_open
    response = urllib.request.urlopen(req)
  File "E:\python\lib\urllib\request.py", line 222, in urlopen
    return opener.open(url, data, timeout)
  File "E:\python\lib\urllib\request.py", line 531, in open
    response = meth(req, response)
  File "E:\python\lib\urllib\request.py", line 641, in http_response
    'http', request, response, code, msg, hdrs)
  File "E:\python\lib\urllib\request.py", line 569, in error
    return self._call_chain(*args)
  File "E:\python\lib\urllib\request.py", line 503, in _call_chain
    result = func(*args)
  File "E:\python\lib\urllib\request.py", line 649, in http_error_default
    raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found

最佳答案

月排行榜 / 总排行榜

Twilight6

2020-8-1 10:43:45

现在煎蛋网不是直接拿数字当页码了，建议你学了正则然后去爬链接吧，这里举个爬一个图库的例子：

1. import re
   
2. import urllib.request
   
3. import os
   
4. 
   
5. 
   
6. def open_url(url):  # 设置访问对象
   
7.     req = urllib.request.Request(url)
   
8.     req.add_header('User-Agent',
   
9.                    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3314.0 Safari/537.36 SE 2.X MetaSr 1.0')
   
10.     response = urllib.request.urlopen(req)
    
11.     html = response.read()
    
12.     return html
    
13. 
    
14. 
    
15. def get_img(html):  # 获取图片
    
16.     p = r'<img src="([^"]+\.jpg)'
    
17.     imglist = re.findall(p, html.decode('utf-8', "replace"))
    
18.     '''
    
19.     for each in imglist:
    
20.         print(each)
    
21.            '''
    
22.     count = 0
    
23.     for each in imglist:
    
24.         count += 1
    
25.         filename = '%d.jpg'
    
26.         pic = 'http:' + each
    
27.         with open(filename % count, 'wb') as f:
    
28.             img = open_url(pic)
    
29.             f.write(img)
    
30. 
    
31. 
    
32. if __name__ == '__main__':
    
33.     url = 'http://jandan.net/ooxx/MjAyMDAxMDMtNjk=#comments'
    
34.     get_img(open_url(url))
    

复制代码

跳转到最佳答案楼层

Twilight6

现在煎蛋网不是直接拿数字当页码了，建议你学了正则然后去爬链接吧，这里举个爬一个图库的例子：

1. import re
   
2. import urllib.request
   
3. import os
   
4. 
   
5. 
   
6. def open_url(url):  # 设置访问对象
   
7.     req = urllib.request.Request(url)
   
8.     req.add_header('User-Agent',
   
9.                    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3314.0 Safari/537.36 SE 2.X MetaSr 1.0')
   
10.     response = urllib.request.urlopen(req)
    
11.     html = response.read()
    
12.     return html
    
13. 
    
14. 
    
15. def get_img(html):  # 获取图片
    
16.     p = r'<img src="([^"]+\.jpg)'
    
17.     imglist = re.findall(p, html.decode('utf-8', "replace"))
    
18.     '''
    
19.     for each in imglist:
    
20.         print(each)
    
21.            '''
    
22.     count = 0
    
23.     for each in imglist:
    
24.         count += 1
    
25.         filename = '%d.jpg'
    
26.         pic = 'http:' + each
    
27.         with open(filename % count, 'wb') as f:
    
28.             img = open_url(pic)
    
29.             f.write(img)
    
30. 
    
31. 
    
32. if __name__ == '__main__':
    
33.     url = 'http://jandan.net/ooxx/MjAyMDAxMDMtNjk=#comments'
    
34.     get_img(open_url(url))
    

复制代码