纵横网址爬小说 Xpath方式下载生成txt文件

capslock133

纵横小说网址http://book.zongheng.com/showchapter/1130229.html 爬取小说 生成txt文件

1. import requests
   
2. import os
   
3. from lxml import etree
   
4. import random
   
5. import time
   
6. import pymysql
   
7. 
   
8. def get_html(url):
   
9.     headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"}
   
10.     res = requests.get(url,headers=headers,timeout=600)
    
11.     res.encoding = res.apparent_encoding
    
12.     #res.encoding = 'UTF-8'
    
13.     #res.encoding='gb2312'
    
14.     htmlmu = res.text
    
15.     return htmlmu
    
16. #获取页面
    
17. def get_book(url):
    
18.     html = get_html(url)
    
19.     selector = etree.HTML(html)
    
20.     nov_name = selector.xpath('///html/body/div[3]/div[1]/h1/text()')  # 标题
    
21.     all_urls = selector.xpath('///html/body/div[3]/div[2]/div[2]/div/ul//a/@href')  # 连接
    
22.     nov_name = ' '.join(nov_name)  # 去除前后[]字符
    
23.     print("匹配到共%d章节" % len(all_urls))
    
24.     k=0
    
25.     for part_url in all_urls:
    
26.         k += 1
    
27.         part_url = f'{part_url}'
    
28.         booktxt=get_html(part_url)
    
29.         selector = etree.HTML(booktxt)
    
30.         title = selector.xpath('//*[@class="title_txtbox"]//text()')  # 标题
    
31.         content = selector.xpath('//*[@class="content"]//text()')  # 作者
    
32.         title = ' '.join(title)  # 去除前后[]字符
    
33.         content = ' '.join(content)  # 去除前后[]字符
    
34.         title = title.strip()
    
35.         content = content.strip()
    
36.         title=f'\n{title}\n\n'
    
37.         content=f'\t{content}\n'
    
38.         content = content.replace("     ", "\n\n  ")
    
39.         new = []
    
40.         new.append(title)
    
41.         new.append(content)
    
42.         down = "./txt/"
    
43.         if os.path.exists(down) != True:
    
44.             os.makedirs(down)
    
45. 
    
46.         print("正在写入第%d章" % k)
    
47.         with open("./txt/" + nov_name + ".txt", 'a+', encoding='utf-8')as p:
    
48.             p.writelines(new)
    
49.         m = random.randint(1,6)
    
50.         time.sleep(m)
    
51. 
    
52. 
    
53. if __name__ == '__main__':   #主函数
    
54.     url="http://book.zongheng.com/showchapter/1130229.html"
    
55.     # url=input("请输入采集目录地址：")
    
56.     get_html(url)
    
57.     get_book(url)

复制代码

capslock133

可以用，好好学习，共同交流。