urllib.error.HTTPError: HTTP Error 403: Forbidden

flybeyondsky

1. import urllib.request
   
2. import os
   
3. 
   
4. def get_page(url):
   
5.     req = urllib.request.Request(url)
   
6.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36')
   
7.     response = urllib.request.urlopen(url)
   
8.     html=response.read().decode('utf-8')
   
9. 
   
10.     a = html.find('current-comment-page') + 23
    
11.     b = html.find(']',a)
    
12. 
    
13.     print(html[a:b])
    
14. 
    
15. 
    
16. def find_imgs(url):
    
17.     pass
    
18. 
    
19. def save_imgs(folder,img_addrs):
    
20.     pass
    
21. 
    
22. def download_mm(folder='OOXX',pages=10):
    
23.     os.mkdir(folder)
    
24.     os.chdir(folder)
    
25. 
    
26.     url = "http://jandan.net/ooxx"
    
27.     page_num = int(get_page(url))
    
28. 
    
29.     for i in range(pages):
    
30.         page_num -= i
    
31.         page_url = url+'page-'+str(page_num)+'#comments'
    
32.         img_addrs = find_imgs(page_url)
    
33.         save_imgs(folder,img_addrs)
    
34. 
    
35. if __name__ == '__main__':
    
36.     download_mm()
    

复制代码

按照小甲鱼的一步一步来，找打印页面的时候 总是报错，urllib.error.HTTPError: HTTP Error 403: Forbidden！应该如何解决？

flybeyondsky

1. import urllib.request
   
2. import os
   
3. import random
   
4. 
   
5. 
   
6. def url_open(url):
   
7.     req = urllib.request.Request(url)
   
8.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36')
   
9. 
   
10. 
    
11.     proxies = ['163.177.79.5:80','124.65.163.10:8080','218.61.39.54:55366']
    
12.     proxy = random.choice(proxies)
    
13.    
    
14.     proxy_support = urllib.request.ProxyHandler({'http':proxy})
    
15.     opener = urllib.request.build_opener(proxy_support)
    
16.     urllib.request.install_opener(opener)
    
17.    
    
18.     response = urllib.request.urlopen(url)
    
19.     html=response.read()
    
20. 
    
21.     return html
    
22. 
    
23. def get_page(url):
    
24.     html=url_open(url).decode('utf-8')
    
25. 
    
26.     a = html.find('current-comment-page') + len('current-comment-page')+3
    
27.     b = html.find(']',a)
    
28. 
    
29.     return html[a:b]
    
30. 
    
31. 
    
32. def find_imgs(url):
    
33.     html = url_open(url).decode('utf-8')
    
34.     img_addrs = []
    
35. 
    
36.     a = html.find('img src=')
    
37.     while a != -1:
    
38.         b = html.find('.jpg',a,a+255)
    
39.         if b != -1:
    
40.             img_addrs.append(html[a+9:b+4])
    
41.         else:
    
42.             b = a + 9
    
43. 
    
44.         a=html.find('img src=',b)
    
45. 
    
46.     return img_addrs
    
47. 
    
48. 
    
49. def save_imgs(folder,img_addrs):
    
50.     for each in img_addrs:
    
51.         filename = each.split('/')[-1]
    
52.         with open(filename,'wb') as f:
    
53.             img=url_open(each)
    
54.             f.wirte(img)
    
55.    
    
56. 
    
57. def download_mm(folder='OOXX',pages=10):
    
58.     os.mkdir(folder)
    
59.     os.chdir(folder)
    
60. 
    
61.     url = "http://jandan.net/ooxx"
    
62.     page_num = int(get_page(url))
    
63. 
    
64.     for i in range(pages):
    
65.         page_num -= i
    
66.         page_url = url+'page-'+str(page_num)+'#comments'
    
67.         img_addrs = find_imgs(page_url)
    
68.         save_imgs(folder,img_addrs)
    
69. 
    
70. if __name__ == '__main__':
    
71.     download_mm()
    

复制代码

加入了代理也不可以哦~~~~

flybeyondsky

1. import urllib.request
   
2. import os
   
3. import random
   
4. 
   
5. 
   
6. def url_open(url):
   
7.     req = urllib.request.Request(url)
   
8.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36')
   
9.     response = urllib.request.urlopen(req)
   
10.     html=response.read()
    
11. 
    
12.     return html
    
13. 
    
14. def get_page(url):
    
15.     html=url_open(url).decode('utf-8')
    
16. 
    
17.     a = html.find('current-comment-page') + len('current-comment-page')+3
    
18.     b = html.find(']',a)
    
19.    
    
20.     return html[a:b]
    
21. 
    
22. 
    
23. def find_imgs(url):
    
24.     html = url_open(url).decode('utf-8')
    
25.     img_addrs = []
    
26. 
    
27.     a = html.find('img src=')
    
28.    
    
29.     while a != -1:
    
30.         b = html.find('.jpg', a, a + 255)
    
31.         if b != -1:
    
32.             img_addrs.append(html[a+9:b+4])
    
33.         else:
    
34.             b = a + 9
    
35. 
    
36.         a=html.find('img src=', b)
    
37. 
    
38.     for each in img_addrs:
    
39.         print(each)
    
40. 
    
41. 
    
42. def save_imgs(folder,img_addrs):
    
43.     for each in img_addrs:
    
44.         filename = each.split('/')[-1]
    
45.         with open(filename,'wb') as f:
    
46.             img=url_open(each)
    
47.             f.wirte(img)
    
48.    
    
49. 
    
50. def download_mm(folder='OOXX',pages=10):
    
51.     os.mkdir(folder)
    
52.     os.chdir(folder)
    
53. 
    
54.     url = 'http://jandan.net/ooxx/'
    
55.     page_num = int(get_page(url))
    
56. 
    
57.     for i in range(pages):
    
58.         page_num -= i
    
59.         page_url = url + 'page-' + str(page_num) + '#comments'
    
60.         print(page_url)
    
61.         img_addrs = find_imgs(page_url)
    
62.         save_imgs(folder,img_addrs)
    
63. 
    
64. if __name__ == '__main__':
    
65.     download_mm()
    

复制代码

出现这样的错误：

http://jandan.net/ooxx/page-1471#comments
http://ww4.sinaimg.cn/mw600/006a7OZUgw1euanawdpncj30dw0dwdh6.jpg
http://ww4.sinaimg.cn/mw600/006a7OZUgw1euanaxrhwfj30dw0dw0u3.jpg
http://ww3.sinaimg.cn/mw600/006a7OZUgw1euanaze0r0j30dw0dw3zw.jpg
http://ww2.sinaimg.cn/mw600/0069EIYdgw1euacgpu1emj30go0m7n06.jpg
http://ww1.sinaimg.cn/mw600/0069EIYdgw1euacgre6sqj30go0m7tat.jpg
http://ww1.sinaimg.cn/mw600/0069EIYdgw1euacguk9ajj30go0m742w.jpg
http://ww4.sinaimg.cn/mw600/0069EIYdgw1euacgwx0x7j30go0m7n1r.jpg
http://ww2.sinaimg.cn/mw600/0069EIYdgw1euacgmiwcvj30go0p2tct.jpg
http://ww1.sinaimg.cn/mw600/8d07dee3tw1eopg9km062j20xc1b4dp7.jpg
http://ww2.sinaimg.cn/mw600/8d07dee3tw1eopg9mm50nj20gm0b7myb.jpg
http://ww1.sinaimg.cn/mw600/df43c939jw1euaixwqnktj20zk1eawqc.jpg
http://ww2.sinaimg.cn/mw600/df43c939jw1euabuk0grnj20f00mita6.jpg
"http://s.jandan.com/static/gggg/56fanli4/"+a+"\" width=\"125\" height=\"125\" border=\"0\" /></a>");

</script><!-- 56fanli -->

                                <script>

var str=new Array("1.jpg
"http://s.jandan.com/static/gggg/niuza/"+a+"\" width=\"125\" height=\"125\" border=\"0\" /></a>");

</script><!-- niuza -->

                                <script>

var str=new Array("1.jpg
Traceback (most recent call last):
  File "C:\Documents and Settings\Administrator\桌面\download_mm.py", line 65, in <module>
    download_mm()
  File "C:\Documents and Settings\Administrator\桌面\download_mm.py", line 62, in download_mm
    save_imgs(folder,img_addrs)
  File "C:\Documents and Settings\Administrator\桌面\download_mm.py", line 43, in save_imgs
    for each in img_addrs:
TypeError: 'NoneType' object is not iterable

@小甲鱼@

tongyangdecuowu,buzhdiaowruhejiejue

longwutian004

听说是被禁止爬虫？

杀手carry

报错是这个     for each in img_addrs:
TypeError: 'NoneType' object is not iterable

请问怎么解决？

杀手carry

是如果列表为None 导致的TypeError: 'NoneType' object is not iterable么

hrbhongwei

urllib.error.HTTPError: HTTP Error 403: Forbidden 加入代理了 还是不可以 这个问题怎么解决呢

wu1996

同样的问题  很无奈

人品弟弟

还是不行，求教/

zrovsion

不知道楼主解决了没