|
|
马上注册,结交更多好友,享用更多功能^_^
您需要 登录 才可以下载或查看,没有账号?立即注册
x
#-*-coding:utf-8
import urllib.request
import re
import random
'''
'''
def open_url(url):
opener = urllib.request.build_opener(proxy_support)
opener.addheader = [('User-Agent' , 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36')]
urllib.request.install_opener(opener)
response = urllib.request.urlopen(url)
html = response.read().decode('utf-8')
return html
def get_img(html):
p = r'(?:(?:[0,1]?\d?\d|2[0-4]\d|25[0-5])\.){3}(?:[0,1]?\d?\d|2[0-4]\d|25[0-5])'
iplist = re.findall(p,html)
return iplist
IPurl = 'https://www.xicidaili.com/nt/'
url = 'https://search.sina.com.cn/?q=习大大&c=news&from=channel&ie=utf-8'
req = urllib.request.Request(IPurl)
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36')
ipweb = urllib.request.urlopen(req)
ip = get_img(ipweb.read().decode('utf-8'))
proxy_support = urllib.request.ProxyHandler({'http':random.choice(ip)})
html = open_url(url)
print(html)
在上述程序在运行过程中总会出现UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd0 in position 400: invalid continuation byte这样的报错,怎么解决啊?是因为不能用utf-8来解码吗 |
|