[已解决]爬虫失败

vonglae

import urllib.request as ure
import re
import os



def open_url(url):
    req = ure.Request(url)
    req.add_header('Referer', url)
    response = ure.urlopen(req)
    html = response.read().decode('uft-8')
    return html


def get_img(html):
    p = r'<img class="BDE_Image" src="([^"]+\.jpg)"'
    img_list = re.findall(p, html)
    temp = []
    for each in img_list:
        temp.append(each)
    return temp


def save_img(folder, img_adders):
    for each in img_adders:
        filename = each.split('/')[-1]
        print(each)
        with open(filename, 'wb') as f:
            img = open_url(each)
            f.write(img)


def tieba(folder='tieba2'):
    os.makedirs(folder, exist_ok=True)
    os.chdir(folder)

    url = 'https://tieba.baidu.com/p/5842501249'
    img_adders = get_img(open_url(url))
    save_img(folder, img_adders)


if __name__ == '__main__':
    tieba()



显示的错误提示为：
Traceback (most recent call last):
  File "C:/Users/Shinelon/Documents/ven1/test.py", line 66, in <module>
    tieba()
  File "C:/Users/Shinelon/Documents/ven1/test.py", line 61, in tieba
    img_adders = get_img(open_url(url))
  File "C:/Users/Shinelon/Documents/ven1/test.py", line 34, in open_url
    html = response.read().decode('uft-8')
LookupError: unknown encoding: uft-8
哪位大佬帮忙看一下QAQ

最佳答案

月排行榜 / 总排行榜

ttyrc33

2018-8-16 22:15:58

1. import urllib.request as ure
   
2. import re
   
3. import os
   
4. 
   
5. 
   
6. 
   
7. def open_url(url):
   
8.     req = ure.Request(url)
   
9.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
   
10.     response = ure.urlopen(req)
    
11.     html = response.read()
    
12.     return html
    
13. 
    
14. 
    
15. def get_img(url):
    
16.     html = open_url(url).decode('utf-8')
    
17.     p = r'<img class="BDE_Image" src="([^"]+\.jpg)"'
    
18.     img_list = re.findall(p, html)
    
19.     temp = []
    
20.     for each in img_list:
    
21.         temp.append(each)
    
22.     return temp
    
23. 
    
24. 
    
25. def save_img(folder, img_adders):
    
26.     for each in img_adders:
    
27.         filename = each.split('/')[-1]
    
28.         print(each)
    
29.         with open(filename, 'wb') as f:
    
30.             img = open_url(each)
    
31.             f.write(img)
    
32. 
    
33. 
    
34. def tieba(folder='tieba2'):
    
35.     os.makedirs(folder, exist_ok=True)
    
36.     os.chdir(folder)
    
37. 
    
38.     url = 'https://tieba.baidu.com/p/5842501249'
    
39.     img_adders = get_img(url)
    
40.     save_img(folder, img_adders)
    
41. 
    
42. 
    
43. if __name__ == '__main__':
    
44.     tieba()
    

复制代码

应该可以了

跳转到最佳答案楼层

塔利班

utf-8

vonglae

塔利班 发表于 2018-8-16 21:39
utf-8

额，我将它改为了utf—8还是同样的报错

vonglae

import urllib.request as ure
import re
import os



def open_url(url):
    req = ure.Request(url)
    req.add_header('Referer', url)
    response = ure.urlopen(req)
    html = response.read().decode('utf-8')
    return html


def get_img(html):
    p = r'<img class="BDE_Image" src="([^"]+\.jpg)"'
    img_list = re.findall(p, html)
    temp = []
    for each in img_list:
        temp.append(each)
    return temp


def save_img(folder, img_adders):
    for each in img_adders:
        filename = each.split('/')[-1]
        print(each)
        with open(filename, 'wb') as f:
            img = open_url(each)
            f.write(img)


def tieba(folder='tieba'):
    os.makedirs(folder, exist_ok=True)
    os.chdir(folder)

    url = 'https://tieba.baidu.com/p/5842501249'
    img_adders = get_img(open_url(url))
    save_img(folder, img_adders)


if __name__ == '__main__':
    tieba()
改为utf-8之后
错误显示为：
Traceback (most recent call last):
  File "C:/Users/Shinelon/Documents/ven1/test.py", line 66, in <module>
    tieba()
  File "C:/Users/Shinelon/Documents/ven1/test.py", line 62, in tieba
    save_img(folder, img_adders)
  File "C:/Users/Shinelon/Documents/ven1/test.py", line 52, in save_img
    img = open_url(each)
  File "C:/Users/Shinelon/Documents/ven1/test.py", line 34, in open_url
    html = response.read().decode('utf-8')
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte

Process finished with exit code 1

塔利班

全改好麻烦，你的错误是因为图片那部分不能decode('utf-8')啊
你参照甲鱼老师视频。decode那部分不放在函数里就行了
打开网页不是下图片数据那部分还得记得加上decode

ttyrc33

1. import urllib.request as ure
   
2. import re
   
3. import os
   
4. 
   
5. 
   
6. 
   
7. def open_url(url):
   
8.     req = ure.Request(url)
   
9.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
   
10.     response = ure.urlopen(req)
    
11.     html = response.read()
    
12.     return html
    
13. 
    
14. 
    
15. def get_img(url):
    
16.     html = open_url(url).decode('utf-8')
    
17.     p = r'<img class="BDE_Image" src="([^"]+\.jpg)"'
    
18.     img_list = re.findall(p, html)
    
19.     temp = []
    
20.     for each in img_list:
    
21.         temp.append(each)
    
22.     return temp
    
23. 
    
24. 
    
25. def save_img(folder, img_adders):
    
26.     for each in img_adders:
    
27.         filename = each.split('/')[-1]
    
28.         print(each)
    
29.         with open(filename, 'wb') as f:
    
30.             img = open_url(each)
    
31.             f.write(img)
    
32. 
    
33. 
    
34. def tieba(folder='tieba2'):
    
35.     os.makedirs(folder, exist_ok=True)
    
36.     os.chdir(folder)
    
37. 
    
38.     url = 'https://tieba.baidu.com/p/5842501249'
    
39.     img_adders = get_img(url)
    
40.     save_img(folder, img_adders)
    
41. 
    
42. 
    
43. if __name__ == '__main__':
    
44.     tieba()
    

复制代码

应该可以了

vonglae

ttyrc33 发表于 2018-8-16 22:15
应该可以了

谢谢大佬