爬取百度百科“猪八戒”词条

lzb1001

import urllib.request
import urllib.parse
import re
from bs4 import BeautifulSoup

def main():
    keyword = input('请输入检索关键词：')
    param = urllib.parse.urlencode({'word':keyword})
    url = 'https://baike.baidu.com/search/word?%s' % param # 或 url = 'https://baike.baidu.com/item/%s' % param[5:]也可

    req = urllib.request.Request(url)
    response = urllib.request.urlopen(req)
    html = response.read().decode('utf-8')
    soup = BeautifulSoup(html, 'html.parser')

    for each in soup.find_all(href = re.compile('item'))[-1]:
        content = ''.join([each.text])
        url2 = ''.join(['http://baike.baidu.com', each['href'] if "%" in each['href'] else urllib.parse.quote(each['href'])]) # 判断是否有进行了 url 编码，没有的话对它进行编码
        req2 = urllib.request.Request(url2)
        response2 = urllib.request.urlopen(req2)
        html2 = response2.read().decode('utf-8')
        soup2 = BeautifulSoup(html2, 'html.parser')
        if soup2.h2:
            content = ''.join([content, soup2.h2.text])
        content = ''.join([content, ' -> ', url2])
        print(content)

if __name__ == '__main__':
    main()

以上代码可以执行，三个问题如下：

1、经测试，为什么以下两个URL好像都可以？一般看哪里&用哪个？
（1）url = '……/item/%s' % param[5:]
（2）url = '……/search/word?%s' % param

2、关于for each in soup.find_all(href = re.compile('item'))[5:-1]:
本想去除运行结果头5个和尾1个无关的结果，所以在代码尾部加上列表分片[5:-1]，但实际仍显示头5个无关结果：

秒懂本尊答目录->……/item/%E7%A7%92%E6%87%82%E6%9C%AC%E5%B0%8A%E7%AD%94
秒懂大师说目录->……/item/%E7%A7%92%E6%87%82%E5%A4%A7%E5%B8%88%E8%AF%B4
秒懂看瓦特目录->……/item/%E7%A7%92%E6%87%82%E7%9C%8B%E7%93%A6%E7%89%B9
秒懂五千年目录->……/item/%E7%A7%92%E6%87%82%E4%BA%94%E5%8D%83%E5%B9%B4
秒懂全视界『寻找世界杯-俄罗斯』特辑->……/item/%E7%A7%92%E6%87%82%E5%85%A8%E8%A7%86%E7%95%8C
……

也就是说在代码尾部加上列表分片[5:-1]，实际仅过滤了下面尾1个无关结果：

本人编辑目录->……/item/%E7%99%BE%E5%BA%A6%E7%99%BE%E7%A7%91%EF%BC%9A%E6%9C%AC%E4%BA%BA%E8%AF%8D%E6%9D%A1%E7%BC%96%E8%BE%91%E6%9C%8D%E5%8A%A1/22442459?bk_fr=pcFooter


所以看来加入列表分片[5:-1]好像只起到了部分作用，但如果：

（1）删除列表分片[5:-1]，程序无法运行并返回以下错误结果：

请输入检索关键词：猪八戒
Traceback (most recent call last):
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 1319, in do_open
    encode_chunked=req.has_header('Transfer-encoding'))
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1252, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1298, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1247, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1026, in _send_output
    self.send(msg)
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 966, in send
    self.connect()
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 938, in connect
    (self.host,self.port), self.timeout, self.source_address)
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\socket.py", line 707, in create_connection
    for res in getaddrinfo(host, port, 0, SOCK_STREAM):
  File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\socket.py", line 752, in getaddrinfo
    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11001] getaddrinfo failed

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
&#160;&#160;File "C:/Users/dell/Desktop/123.py", line 30, in <module>
&#160; &#160; main()
&#160;&#160;File "C:/Users/dell/Desktop/123.py", line 21, in main
&#160; &#160; response2 = urllib.request.urlopen(req2)
&#160;&#160;File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 222, in urlopen
&#160; &#160; return opener.open(url, data, timeout)
&#160;&#160;File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 525, in open
&#160; &#160; response = self._open(req, data)
&#160;&#160;File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 543, in _open
&#160; &#160; '_open', req)
&#160;&#160;File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 503, in _call_chain
&#160; &#160; result = func(*args)
&#160;&#160;File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 1347, in http_open
&#160; &#160; return self.do_open(http.client.HTTPConnection, req)
&#160;&#160;File "C:\Users\dell\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 1321, in do_open
&#160; &#160; raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 11001] getaddrinfo failed>

（2）将列表分片修改为[:-1]，则返回如下错误提示：
请输入检索关键词：猪八戒
Traceback (most recent call last):
  File "D:\work\p14_92.py", line 35, in <module>
    main()
  File "D:\work\p14_92.py", line 24, in main
    url2 = ''.join(['http://baike.baidu.com', each['href'] if "%" in each['href'] else urllib.parse.quote(each['href'])]) # 判断是否有进行了 url 编码，没有的话对它进行编码
TypeError: string indices must be integers

3、如果要加入伪装的header，应如何添加？

suchocolate

爬取词条的什么，把目标明确的说明一下。