爬虫防屏蔽问题

喵爷

源码：
import urllib.request
def use_proxy(url,proxy_addr):
    proxy = urllib.request.ProxyHandler({"http":proxy_addr})
    opener = urllib.request.build_opener(proxy, urllib.request.HTTPHandler)
    urllib.request.install_opener(opener)
    data = urllib.request.urlopen(url).read().decode("utf-8","ignore")
    return data

proxy_addr = "117.91.132.6:9999"
url = "http://www.baidu.com"
data = use_proxy(url, proxy_addr)
print(len(data))

报错：
Traceback (most recent call last):
  File "/usr/lib/python3.6/urllib/request.py", line 1318, in do_open
    encode_chunked=req.has_header('Transfer-encoding'))
  File "/usr/lib/python3.6/http/client.py", line 1239, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
    self.send(msg)
  File "/usr/lib/python3.6/http/client.py", line 964, in send
    self.connect()
  File "/usr/lib/python3.6/http/client.py", line 936, in connect
    (self.host,self.port), self.timeout, self.source_address)
  File "/usr/lib/python3.6/socket.py", line 724, in create_connection
    raise err
  File "/usr/lib/python3.6/socket.py", line 713, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/fish/Ubuntu_script/The crawler/demo7.py", line 11, in <module>
    data = use_proxy(url, proxy_addr)
  File "/home/fish/Ubuntu_script/The crawler/demo7.py", line 6, in use_proxy
    data = urllib.request.urlopen(url).read().decode("utf-8","ignore")
  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
    response = self._open(req, data)
  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
    '_open', req)
  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
    result = func(*args)
  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
    return self.do_open(http.client.HTTPConnection, req)
  File "/usr/lib/python3.6/urllib/request.py", line 1320, in do_open
    raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 111] Connection refused>

Process finished with exit code 1

zltzlt

网站识别出那是一只爬虫，要用代理

害怕型全嘲豪

zltzlt 发表于 2019-7-27 11:57
网站识别出那是一只爬虫，要用代理

他不是已经用了代理了么？ 他这是因为Headers没设置 。 而且并不推荐用 urllib 用 request 简单方便快捷。

害怕型全嘲豪

而且百度要用 https 不是 http