爬取起点免费作品

小wu

自己模仿别人的写的，练练手
起点中文网免费专栏爬取
使用模块：requests，re

1. import requests
   
2. import re
   
3. import time
   
4. 
   
5. 
   
6. url = 'https://www.qidian.com/free/all/'
   
7. book_img_box = requests.get(url=url)
   
8. 
   
9. def link_list():
   
10.     list_page = requests.get(url=href)
    
11.     info_page = re.findall('<h2 class="book_name"><a href="(.*?)" target="_blank" data-eid="qd_G55" data-cid=".*?" alt="(.*?)" title=".*?</a></h2>', list_page.text)
    
12.     for link, title in info_page:
    
13.         link = 'https:' + link
    
14.         print(link, title)
    
15.         #time.sleep(1)
    
16.         response = requests.get(url=link)
    
17.         content_data = re.findall('window.cContent = "(.*?)";', response.text, re.S)[0]
    
18.         text = title + '\n' + content_data.replace('&lt;p&gt;\u3000\u3000', '') + '\n'
    
19.         print(text)
    
20.         with open(alt + '.txt', mode='a', encoding='utf-8') as f:
    
21.             f.write(text)
    
22. 
    
23. book_name = re.findall('<h2><a href="(.*?)".*?title=".*?">(.*?)</a></h2>', book_img_box.text)
    
24. for href, alt in book_name:
    
25.     href = 'https:' + href
    
26.     print(href, alt)
    
27.     link_list()
    

复制代码