为什么加上add_header就不行了

幽梦三影

1. from bs4 import BeautifulSoup
   
2. import urllib.request
   
3. import re
   
4. 
   
5. def get_html(url):
   
6.     rep = urllib.request.Request(url)
   
7.     #rep.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/61.0')
   
8.     html = urllib.request.urlopen(rep).read().decode('utf-8')
   
9.     return html
   
10. 
    
11. url = 'https://baike.baidu.com/item/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB'
    
12. html = get_html(url)
    
13. s = BeautifulSoup(html,'lxml')
    
14. link = s.find_all('a',href = re.compile('/item'))
    
15. count = 0
    
16. for i in link[:3]:
    
17.     l = get_html('https://baike.baidu.com' + i['href'])
    
18.     with open('%d.txt'%count,'w',encoding = 'utf-8') as f:
    
19.         f.write(l)
    
20.     count += 1
    
21. 
    
22. 
    

复制代码

ba21

我敢保证，rep.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/61.0')这里面有   中文字符   自己慢慢找，琢磨吧