爬取起点免费作品

小wu

自己模仿别人的写的，练练手
起点中文网免费专栏爬取
使用模块：requests，re

import requests
import re
import time


url = 'https://www.qidian.com/free/all/'
book_img_box = requests.get(url=url)

def link_list():
    list_page = requests.get(url=href)
    info_page = re.findall('<h2 class="book_name"><a href="(.*?)" target="_blank" data-eid="qd_G55" data-cid=".*?" alt="(.*?)" title=".*?</a></h2>', list_page.text)
    for link, title in info_page:
        link = 'https:' + link
        print(link, title)
        #time.sleep(1)
        response = requests.get(url=link)
        content_data = re.findall('window.cContent = "(.*?)";', response.text, re.S)[0]
        text = title + '\n' + content_data.replace('<p>\u3000\u3000', '') + '\n'
        print(text)
        with open(alt + '.txt', mode='a', encoding='utf-8') as f:
            f.write(text)

book_name = re.findall('<h2><a href="(.*?)".*?title=".*?">(.*?)</a></h2>', book_img_box.text)
for href, alt in book_name:
    href = 'https:' + href
    print(href, alt)
    link_list()