纵横网址爬小说 Xpath方式下载生成txt文件

capslock133

纵横小说网址http://book.zongheng.com/showchapter/1130229.html 爬取小说 生成txt文件

import requests
import os
from lxml import etree
import random
import time
import pymysql

def get_html(url):
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"}
    res = requests.get(url,headers=headers,timeout=600)
    res.encoding = res.apparent_encoding
    #res.encoding = 'UTF-8'
    #res.encoding='gb2312'
    htmlmu = res.text
    return htmlmu
#获取页面
def get_book(url):
    html = get_html(url)
    selector = etree.HTML(html)
    nov_name = selector.xpath('///html/body/div[3]/div[1]/h1/text()')  # 标题
    all_urls = selector.xpath('///html/body/div[3]/div[2]/div[2]/div/ul//a/@href')  # 连接
    nov_name = ' '.join(nov_name)  # 去除前后[]字符
    print("匹配到共%d章节" % len(all_urls))
    k=0
    for part_url in all_urls:
        k += 1
        part_url = f'{part_url}'
        booktxt=get_html(part_url)
        selector = etree.HTML(booktxt)
        title = selector.xpath('//*[@class="title_txtbox"]//text()')  # 标题
        content = selector.xpath('//*[@class="content"]//text()')  # 作者
        title = ' '.join(title)  # 去除前后[]字符
        content = ' '.join(content)  # 去除前后[]字符
        title = title.strip()
        content = content.strip()
        title=f'\n{title}\n\n'
        content=f'\t{content}\n'
        content = content.replace("     ", "\n\n  ")
        new = []
        new.append(title)
        new.append(content)
        down = "./txt/"
        if os.path.exists(down) != True:
            os.makedirs(down)

        print("正在写入第%d章" % k)
        with open("./txt/" + nov_name + ".txt", 'a+', encoding='utf-8')as p:
            p.writelines(new)
        m = random.randint(1,6)
        time.sleep(m)


if __name__ == '__main__':   #主函数
    url="http://book.zongheng.com/showchapter/1130229.html"
    # url=input("请输入采集目录地址：")
    get_html(url)
    get_book(url)

capslock133

可以用，好好学习，共同交流。