爬虫爬出来的网页是空白的，求纠错

香辣烤鱿鱼

以下代码的作用是 在百度贴吧输入搜索内容然后爬取其中的网页，但保存的文件是空白，求纠错，谢谢。
注：需要填搜索内容，跟页数，如内容是“苹果”，页数是“5”


from urllib.request import Request,urlopen
from urllib.parse import urlencode
from fake_useragent import UserAgent
import re

def get_html(url):
    headers = {
        "User-Agent": UserAgent().chrome
    }
    request = Request(url,headers = headers)
    response = urlopen(request)
    print(response.read().decode())
    return response.read()

def save_html(filename,html_bytes):
    with open(filename,"wb")as f:
        f.write(html_bytes)

def main():
    content = input("请输入要下载的内容：")
    num = input("请输入要下载多少页：")
    base_url = "http://tieba.baidu.com/f?ie=utf-8&{}"
    for pn in range(int(num)):
        args = {
            "pn": pn * 50,
            "kw": content
        }
        filename = "第" + str(pn + 1) + "页.html"
        args = urlencode(args)
        print("正在保存"+ filename)
        html_bytes = get_html(base_url.format(args))
        save_html(filename,html_bytes)


if __name__ == "__main__":
    main()

天羽妖月

改了个保存方式就好了

from urllib.request import Request, urlopen
from urllib.parse import urlencode
from fake_useragent import UserAgent
import re


def get_html(url):
    headers = {
        "User-Agent": 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
    }
    request = Request(url, headers=headers)
    response = urlopen(request)
    # print(response.read().decode())
    return response.read().decode()


def save_html(filename, html_bytes):
    with open(filename, "w")as f:
        f.write(html_bytes)


def main():
    content = input("请输入要下载的内容：")
    num = input("请输入要下载多少页：")
    base_url = "http://tieba.baidu.com/f?ie=utf-8&{}"
    # url = 'http://tieba.baidu.com/f?kw=%E8%8B%B9%E6%9E%9C&ie=utf-8&pn=50'
    for pn in range(int(num)):
        args = {
            "pn": pn * 50,
            "kw": content
        }
        filename = "第" + str(pn + 1) + "页.html"
        args = urlencode(args)
        print("正在保存" + filename)
        html_bytes = get_html(base_url)
        save_html(filename, html_bytes)
        # print(html_bytes.decode())
        # with open('1.html', 'w') as f:
        #     f.write(html_bytes.decode())

if __name__ == "__main__":
    main()

香辣烤鱿鱼

天羽妖月 发表于 2019-3-26 15:08
改了个保存方式就好了

from urllib.request import Request, urlopen

报错呢。。

Traceback (most recent call last):
  File "D:/Interest/Python/学习02_爬虫_20190316/day01/test.py", line 42, in <module>
    main()
  File "D:/Interest/Python/学习02_爬虫_20190316/day01/test.py", line 36, in main
    save_html(filename, html_bytes)
  File "D:/Interest/Python/学习02_爬虫_20190316/day01/test.py", line 19, in save_html
    f.write(html_bytes)
UnicodeEncodeError: 'gbk' codec can't encode character '\u2022' in position 74816: illegal multibyte sequence

小小.甲鱼

1. from urllib.request import Request, urlopen
   
2. from urllib.parse import urlencode
   
3. from fake_useragent import UserAgent
   
4. import re
   
5. 
   
6. 
   
7. def decode(data):
   
8.     if isinstance(data, bytes):
   
9.         try:
   
10.             return data.decode('utf-8')
    
11.         except Exception:
    
12.             return data.decode('gb18030')
    
13.     return data
    
14. 
    
15. 
    
16. def get_html(url):
    
17.     headers = {
    
18.         "User-Agent": UserAgent().chrome
    
19.     }
    
20.     request = Request(url, headers=headers)
    
21.     response = urlopen(request)
    
22.     return response.read()
    
23. 
    
24. 
    
25. def save_html(filename, html_bytes):
    
26.     try:
    
27.         html = decode(html_bytes)
    
28.     except Exception as e:
    
29.         print('解码失败!')
    
30.         return
    
31. 
    
32.     with open(filename, "w") as f:
    
33.         f.write(html)
    
34. 
    
35. 
    
36. def main():
    
37.     content = input("请输入要下载的内容：")
    
38.     num = input("请输入要下载多少页：")
    
39.     base_url = "http://tieba.baidu.com/f?ie=utf-8&{}"
    
40.     for pn in range(int(num)):
    
41.         args = {
    
42.             "pn": pn * 50,
    
43.             "kw": content
    
44.         }
    
45.         filename = "第" + str(pn + 1) + "页.html"
    
46.         args = urlencode(args)
    
47.         print("正在保存" + filename)
    
48.         html_bytes = get_html(base_url.format(args))
    
49.         save_html(filename, html_bytes)
    
50. 
    
51. 
    
52. if __name__ == "__main__":
    
53.     main()
    

复制代码

香辣烤鱿鱼

小小.甲鱼 发表于 2019-3-27 18:59

谢谢，有提示编码错误，但将 “with open(filename, "w") as f:”修改为“with open(filename, "w",encoding="utf-8"”就好了