爬取小说（python3.7.0, urllib, xpath, python自带的模块）

金刚

1. # 网站地址：https://www.biqiuge.com/     （笔趣阁）
   
2. 
   
3. from urllib.request import Request, urlopen
   
4. from urllib.error import URLError
   
5. from lxml import etree
   
6. from time import sleep
   
7. import os, re, random
   
8. 
   
9. 
   
10. # 获取html
    
11. def get_html(url):
    
12. 
    
13.         headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'}
    
14.         req = Request(url, headers=headers)
    
15.         try:
    
16.                 response = urlopen(req)
    
17.         except URLError as e:
    
18.                 if hasattr(e, "reason"):
    
19.                         print("We failed to reach a server.")
    
20.                         print("Reason:", e.reason)
    
21.                 elif hasattr(e, "code"):
    
22.                         print("The server couldn\'t fulfill the request.")
    
23.                         print("Error code:", e.code)
    
24.         else:
    
25.                 html = response.read()
    
26.         return html
    
27. 
    
28. 
    
29. def get_all_chapter(html):
    
30. 
    
31.        
    
32.         base_url = "https://www.biqiuge.com"
    
33.         start = int(input("请输入开始章数："))
    
34.         end = int(input("请输入结束章数:"))
    
35. 
    
36.         html_x = etree.HTML(html)
    
37.         chapter_title = html_x.xpath('//div[@class="listmain"]//dd[position() > 6]/a/text()')
    
38.         chapter_title = chapter_title[start-1:end]
    
39. 
    
40.         all_chapter = html_x.xpath('//div[@class="listmain"]//dd[position() > 6]/a/@href')
    
41.         #列表推导式 完整化章节url
    
42.         each_chapter_url = [base_url+suffix for suffix in all_chapter]
    
43.         each_chapter_url = each_chapter_url[start-1:end]
    
44.         return [each_chapter_url,chapter_title]
    
45.        
    
46. 
    
47. 
    
48. # 获取章节内容并保存
    
49. def get_content_and_save(all_chapter):
    
50.        
    
51.         files = input("请输入保存小说的文件夹名称：")
    
52.         os.mkdir(files)
    
53.         counter = 0
    
54.         for each_chapter_url in all_chapter[0]:
    
55. 
    
56.                 html = get_html(each_chapter_url)
    
57.                 html = html.decode("gbk")
    
58.                 tree = etree.HTML(html)
    
59.                 # 获取章节内容到列表
    
60.                 fiction_content = tree.xpath('//div[@id="content"]/text()')
    
61.                 # 优化获取章节内容
    
62.                 text_content = ""
    
63.                 for each_line_content in fiction_content:
    
64.                         text_content += each_line_content
    
65.                 beautiful_text = text_content.replace("\r", "\n").replace(" ","")
    
66.                 # 保存
    
67.                 print("正在下载：", all_chapter[1][counter])
    
68.                 with open("./" + files + "/" + all_chapter[1][counter] + ".txt", "w", encoding="utf-8") as f:
    
69.                         f.write(beautiful_text)
    
70.                 counter += 1
    
71.                 wait_time = random.choice([3, 4, 5, 6])
    
72.                 sleep(wait_time)
    
73. 
    
74. # 主函数
    
75. def get_fictions(url):
    
76. 
    
77.         html = get_html(url)
    
78.         html = html.decode("gbk")
    
79.         all_chapter = get_all_chapter(html)
    
80.         get_content_and_save(all_chapter)
    
81.        
    
82. 
    
83. if __name__ == "__main__":
    
84. 
    
85.         url = input("请输入小说章节页面地址：") # 例如下面格式的url
    
86.         # url = "https://www.biqiuge.com/book/24277/"
    
87.         # url = "https://www.biqiuge.com/book/4772/"
    
88.         get_fictions(url)
    
89.         print("已停止下载！")

复制代码