爬虫编码

ZQLSTC

1. import urllib.request
   
2. import os
   
3. 
   
4. def url_open(url):
   
5.     res = urllib.request.Request(url)
   
6.     res.add_header("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36")
   
7.     response = urllib.request.urlopen(res)
   
8.     html = response.read()
   
9.     return html
   
10. 
    
11. def find_img(url):
    
12.     html = url_open(url).decode("utf-8")
    
13.     a = html.find("img src=")
    
14.     img_addrs=[]
    
15.     while a!=-1:
    
16.         b = html.find('id="bigpicimg"',a,a+255)
    
17.         if b==-1:
    
18.             img_addrs.append(html[a+9:b-2])
    
19.         else :
    
20.             b=a
    
21.         a = html.find("img src=",b)
    
22.         
    
23.     for i in img_addrs:
    
24.         print(i)
    
25.    
    
26. 
    
27. 
    
28. def download(folder = "chengduling",page = 5):
    
29.     os.mkdir(folder)
    
30.     os.chdir(folder)
    
31. 
    
32.     url="http://www.tupianzj.com/mingxing/xiezhen/20180625/"
    
33.     for i in range(1,6):
    
34.         if i==1:
    
35.             page_num = "163771"
    
36.         else:
    
37.             page_num = "163771"+"_"+str(i)
    
38.         page_url = url + page_num + ".html"
    
39.         img_url = find_img(page_url)
    
40. 
    
41. 
    
42. if __name__ == "__main__":
    
43.     download()
    
44. 
    

复制代码

1. Traceback (most recent call last):
   
2.   File "C:/Users/123/Desktop/chengduling.py", line 43, in <module>
   
3.     download()
   
4.   File "C:/Users123/Desktop/chengduling.py", line 39, in download
   
5.     img_url = find_img(page_url)
   
6.   File "C:/Users/123/Desktop/chengduling.py", line 12, in find_img
   
7.     html = url_open(url).decode("utf-8")
   
8. UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb3 in position 309: invalid start byte

复制代码

编码报错要怎么解决

拉了盏灯

应该是有中文，decode('gbk')试试

我爱鱼C！

html = url_open(url).decode("GB2312")
改成这个试试

小小小小的鱼丶

1. def url_open(url):
   
2.     res = urllib.request.Request(url)
   
3.     res.add_header("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36")
   
4.     response = urllib.request.urlopen(res)
   
5. #这里加一句
   
6.    response.encoding='utf-8'
   
7.     html = response.read()
   
8.     return html

复制代码