爬取百度百科“猪八戒”词条
import urllib.requestimport urllib.parse
import re
from bs4 import BeautifulSoup
def main():
keyword = input('请输入检索关键词：')
param = urllib.parse.urlencode({'word':keyword})
url = 'https://baike.baidu.com/search/word?%s' % param # 或 url = 'https://baike.baidu.com/item/%s' % param也可
req = urllib.request.Request(url)
response = urllib.request.urlopen(req)
html = response.read().decode('utf-8')
soup = BeautifulSoup(html, 'html.parser')
for each in soup.find_all(href = re.compile('item'))[-1]:
content = ''.join()
url2 = ''.join(['http://baike.baidu.com', each['href'] if "%" in each['href'] else urllib.parse.quote(each['href'])]) # 判断是否有进行了 url 编码，没有的话对它进行编码
req2 = urllib.request.Request(url2)
response2 = urllib.request.urlopen(req2)
html2 = response2.read().decode('utf-8')
soup2 = BeautifulSoup(html2, 'html.parser')
if soup2.h2:
content = ''.join()
content = ''.join()
print(content)
if __name__ == '__main__':
main()
以上代码可以执行，三个问题如下：
1、经测试，为什么以下两个URL好像都可以？一般看哪里&用哪个？
（1）url = '……/item/%s' % param
（2）url = '……/search/word?%s' % param
2、关于for each in soup.find_all(href = re.compile('item')):
本想去除运行结果头5个和尾1个无关的结果，所以在代码尾部加上列表分片，但实际仍显示头5个无关结果：
秒懂本尊答目录->……/item/%E7%A7%92%E6%87%82%E6%9C%AC%E5%B0%8A%E7%AD%94
秒懂大师说目录->……/item/%E7%A7%92%E6%87%82%E5%A4%A7%E5%B8%88%E8%AF%B4
秒懂看瓦特目录->……/item/%E7%A7%92%E6%87%82%E7%9C%8B%E7%93%A6%E7%89%B9
秒懂五千年目录->……/item/%E7%A7%92%E6%87%82%E4%BA%94%E5%8D%83%E5%B9%B4
秒懂全视界『寻找世界杯-俄罗斯』特辑->……/item/%E7%A7%92%E6%87%82%E5%85%A8%E8%A7%86%E7%95%8C
……
也就是说在代码尾部加上列表分片，实际仅过滤了下面尾1个无关结果：
本人编辑目录->……/item/%E7%99%BE%E5%BA%A6%E7%99%BE%E7%A7%91%EF%BC%9A%E6%9C%AC%E4%BA%BA%E8%AF%8D%E6%9D%A1%E7%BC%96%E8%BE%91%E6%9C%8D%E5%8A%A1/22442459?bk_fr=pcFooter
所以看来加入列表分片好像只起到了部分作用，但如果：
（1）删除列表分片，程序无法运行并返回以下错误结果：
请输入检索关键词：猪八戒
Traceback (most recent call last):
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 1319, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 966, in send
self.connect()
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 938, in connect
(self.host,self.port), self.timeout, self.source_address)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\socket.py", line 707, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\socket.py", line 752, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/dell/Desktop/123.py", line 30, in <module>
main()
File "C:/Users/dell/Desktop/123.py", line 21, in main
response2 = urllib.request.urlopen(req2)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 525, in open
response = self._open(req, data)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 543, in _open
'_open', req)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 503, in _call_chain
result = func(*args)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 1347, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 1321, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error getaddrinfo failed>
（2）将列表分片修改为[:-1]，则返回如下错误提示：
请输入检索关键词：猪八戒
Traceback (most recent call last):
File "D:\work\p14_92.py", line 35, in <module>
main()
File "D:\work\p14_92.py", line 24, in main
url2 = ''.join(['http://baike.baidu.com', each['href'] if "%" in each['href'] else urllib.parse.quote(each['href'])]) # 判断是否有进行了 url 编码，没有的话对它进行编码
TypeError: string indices must be integers
3、如果要加入伪装的header，应如何添加？ 爬取词条的什么，把目标明确的说明一下。
页:
[1]