[爬虫] urllib使用ProxyHandler时可缺省参数不使用代理
本帖最后由 591821661 于 2021-10-5 08:20 编辑同样的代码
urllib被远程拒绝 (经常抽风，有时又可以，感兴趣的可以试试)
requests一直能用
import urllib.request
import json
import sys
import requests
def useproxy(proxy='127.0.0.1:80'):
proxy_support = urllib.request.ProxyHandler({'http': proxy})
opener = urllib.request.build_opener(proxy_support)
opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'),
('Host','ip-api.com'),
('Connection','keep-alive'),
('Upgrade-Insecure-Requests',1),
('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'),
('Accept-Encoding','gzip, deflate'),
('Accept-Language','zh-CN,zh;q=0.9'),
]
urllib.request.install_opener(opener)
def get_ip():
# Get IP Address
try:
ip_api_url = 'http://ip-api.com/json/?fields=61439'
ip_result = urllib.request.urlopen(ip_api_url)
proxy_text = ip_result.read().decode('utf-8')
s_get_dict = json.loads(proxy_text)
print('----------------------IP Information Begin--------------------------')
for key in s_get_dict:
print('%s : %s' % (key, s_get_dict))
print('----------------------IP Information End--------------------------')
except:
print('url_get error')
print(sys.exc_info(), sys.exc_info())
ip_api_url = 'http://ip-api.com/json/?fields=61439'
#useproxy()
get_ip()
#ip_result = urllib.request.urlopen(ip_api_url)
#s = requests.get(ip_api_url)
问题已经解决：
proxy_handler = urllib.request.ProxyHandler({})#正确写法
proxy_handler = urllib.request.ProxyHandler(
{
'http':'127.0.0.1:80',
'https':'127.0.0.1:80',
}
) 哈哈哈啊哈哈哈哈哈哈哈{:10_250:} 我感觉楼主在开车，但是我没有证据
标题可以，哈哈
urllib：小瓶张嘎
requests：举起酒来
本帖最后由 591821661 于 2021-4-27 21:14 编辑
我不是水贴啊，分享干货{:10_254:}！
找到为啥时好时坏的原因了！！{:10_279:}
一句话总结：
不要在ProxyHandler中使用127.0.0.1:80
因为之前有项目用了127.0.0.1:80 能正常运行 就没往这个方向考虑！
看来这个网站比较皮哦
proxy_handler = urllib.request.ProxyHandler({})正确写法
proxy_handler = urllib.request.ProxyHandler(
{
'http':'127.0.0.1:80',
'https':'127.0.0.1:80',
}
)
错误示范
测试代码
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 27 19:32:49 2021
@author: 591821661
"""
import urllib.request
import json
import sys
def useproxy(proxy=''):
if proxy:
proxy_support = urllib.request.ProxyHandler({
'http': proxy,
'https' : proxy,
})
else:
proxy_support = urllib.request.ProxyHandler({
})
opener = urllib.request.build_opener(proxy_support)
opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'),
# ('Host','ip-api.com'),
# ('Connection','keep-alive'),
# ('Upgrade-Insecure-Requests',1),
# ('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'),
# ('Accept-Encoding','gzip, deflate'),
# ('Accept-Language','zh-CN,zh;q=0.9'),
]
urllib.request.install_opener(opener)
def get_ip():
# Get IP Address
try:
ip_api_url = 'http://ip-api.com/json/?fields=61439'
ip_result = urllib.request.urlopen(ip_api_url)
proxy_text = ip_result.read().decode('utf-8')
s_get_dict = json.loads(proxy_text)
print('----------------------IP Information Begin--------------------------')
for key in s_get_dict:
print('%s : %s' % (key, s_get_dict))
print('----------------------IP Information End--------------------------')
except:
print('url_get error')
print(sys.exc_info(), sys.exc_info())
if __name__ == '__main__':# 如果运行该文件，而不是作为库函数导入 则运行下列代码
ip_api_url = 'http://ip-api.com/json/?fields=61439'
useproxy()# 正常
#useproxy('127.0.0.1:80')# 会报错
get_ip()
页:
[1]