[已解决]爬取ip地址出现问题

Chysial

如果url一页一页输入没有问题，加入循环就出现问题了：
url:https://www.kuaidaili.com/free/inha/3   每页就是最后的页码改变，其余的不变

1. import urllib.request
   
2. import re
   
3. import os
   
4. import random
   
5. def url_open(url):
   
6.     req = urllib.request.Request(url)
   
7.     req.add_header("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36")
   
8.     proxies = ["125.108.123.66:9000","183.166.103.45:9999","115.195.84.31:8118","61.178.149.237:59042"]
   
9.     proxy = random.choice(proxies)
   
10.     proxy_support = urllib.request.ProxyHandler({'http':proxy})
    
11.     opener = urllib.request.build_opener(proxy_support)
    
12.     urllib.request.install_opener(opener)
    
13.     response = urllib.request.urlopen(req)
    
14.     html = response.read().decode("utf-8")
    
15.     return html
    
16. def split_addrs_port(temp_str):
    
17.     [a,b] = temp_str.split(">",2)
    
18.     [c,d] =b.split("<",2)
    
19.     return c
    
20.    
    
21. def get_addrs_port(url):
    
22.     html = url_open(url)
    
23.     compile_ip = re.compile(r'("IP">(([0-1]?\d?\d|2[0-4]\d|25[0-5])\.){3}([0-1]?\d?\d|2[0-4]\d|25[0-5])<)')
    
24.     compile_port = re.compile(r""PORT">\d{0,5}<")
    
25.     url_addrs_temp = compile_ip.findall(html)
    
26.     url_port_temp= compile_port.findall(html)
    
27.     list_addrs_port = []
    
28.     for i in range(len(url_addrs_temp)):
    
29.         url_addrs = split_addrs_port(str(url_addrs_temp[i][0]))
    
30.         url_port = split_addrs_port(str(url_port_temp[i]))
    
31.         url_addrs_port = url_addrs + ":" + url_port
    
32.         list_addrs_port.append(url_addrs_port)
    
33.     return list_addrs_port
    
34. def save_addrs_port(addrs_port):
    
35.     with open("ip.docx","a") as f:
    
36.         for each in addrs_port:
    
37.             f.write(str(each) + "\n")
    
38.    
    
39. 
    
40. def download_url(folder = "proxy_support", pages = "2"):
    
41.     '''os.chdir("C:\\Users\\Chysial\\Desktop")
    
42.     os.mkdir(folder)
    
43.     os.chdir(folder)'''
    
44. 
    
45. 
    
46.     for i in range(int(pages)):
    
47.         page_num = i+1
    
48.         url = "https://www.kuaidaili.com/free/inha/"
    
49.         pages_url = url +str(page_num)
    
50.         url_addrs_port = get_addrs_port(pages_url)
    
51.         '''save_addrs_port(url_addrs_port)'''
    
52.         print(url_addrs_port)
    
53. 
    
54. if __name__ == "__main__":
    
55.     download_url()
    

复制代码

结果：
['36.249.48.41:9999', '113.124.92.190:9999', '113.195.202.209:9999', '163.204.245.192:9999', '220.249.149.241:9999', '115.218.2.38:9000', '220.176.33.151:9999', '58.32.34.219:8118', '123.163.96.8:9999', '116.196.87.86:20183', '113.120.35.180:9999', '182.149.83.171:9999', '111.160.169.54:41820', '118.113.246.159:9999', '60.13.42.67:9999']
Traceback (most recent call last):
  File "C:/python/5.爬虫/proxy_support.py", line 55, in <module>
    download_url()
  File "C:/python/5.爬虫/proxy_support.py", line 50, in download_url
    url_addrs_port = get_addrs_port(pages_url)
  File "C:/python/5.爬虫/proxy_support.py", line 22, in get_addrs_port
    html = url_open(url)
  File "C:/python/5.爬虫/proxy_support.py", line 13, in url_open
    response = urllib.request.urlopen(req)
  File "C:\Users\Chysial\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\Chysial\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 532, in open
    response = meth(req, response)
  File "C:\Users\Chysial\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 642, in http_response
    'http', request, response, code, msg, hdrs)
  File "C:\Users\Chysial\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 570, in error
    return self._call_chain(*args)
  File "C:\Users\Chysial\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 504, in _call_chain
    result = func(*args)
  File "C:\Users\Chysial\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 650, in http_error_default
    raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 503: Service Temporarily Unavailable

我找了半天，不知道哪里出错了，各位大佬看看啊!(注，我那正则表达式有点问题，没有取消贪婪，这样对结果没影响，各位大佬先帮忙找一下，为啥出现错误吧）

最佳答案

月排行榜 / 总排行榜

陈尚涵

2020-8-18 18:39:53

Chysial 发表于 2020-3-13 19:49
也就是我的代码基本没啥问题了？

自己去这个网页看哈

https://man.ilovefishc.com/pageHTML5/HTTP.html

你的代码确实没问题，但是由于超载或系统维护，服务器暂时的无法处理客户端的请求。延时的长度在服务器的Retry-After头信息中

跳转到最佳答案楼层

Chysial

@塔利班 @claws0n

塔利班

Chysial 发表于 2020-3-13 18:04
@塔利班 @claws0n

503不是服务端的问题么

Chysial

塔利班 发表于 2020-3-13 19:12
503不是服务端的问题么

也就是我的代码基本没啥问题了？

陈尚涵

Chysial 发表于 2020-3-13 19:49
也就是我的代码基本没啥问题了？

自己去这个网页看哈

https://man.ilovefishc.com/pageHTML5/HTTP.html

你的代码确实没问题，但是由于超载或系统维护，服务器暂时的无法处理客户端的请求。延时的长度在服务器的Retry-After头信息中

Chysial

陈尚涵 发表于 2020-8-18 18:39
自己去这个网页看哈

https://man.ilovefishc.com/pageHTML5/HTTP.html

我这个多久的帖子了，谢谢了

陈尚涵

Chysial 发表于 2020-8-19 09:54
我这个多久的帖子了，谢谢了

不用谢，都是应该的