萌新的爬虫——爬取斗罗小说

tRNA

直接上代码：

1. import requests
   
2. import time
   
3. import re
   
4. import os
   
5. from bs4 import BeautifulSoup
   
6. 
   
7. path = "D:/APython/spider/douluo/斗罗大陆3龙王传说/"# 路径记得改
   
8. url = "https://www.bifeige.com/9_9235/"
   
9. 
   
10. def openurl(url):
    
11.     headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36 Edg/88.0.705.81"}
    
12.     respound = requests.get(url, headers=headers)
    
13.     respound.encoding = respound.apparent_encoding
    
14.     return respound.text
    
15. 
    
16. def save_text(respound, name):
    
17.     # 解析并保存文件
    
18.     try:
    
19.         soup = BeautifulSoup(respound, "html.parser")
    
20.         targets = soup.find_all("div", id="content")
    
21.         with open(path + name, "a", encoding="utf-8") as f:
    
22.             for target in targets:
    
23.                 f.write(target.text)
    
24.                 print("下载成功；" + name)
    
25.     except OSError:
    
26.         return ""
    
27. 
    
28. respound = openurl(url)
    
29. soup = BeautifulSoup(respound, "html.parser")
    
30. targets = soup.find_all("dd")
    
31. i = 1
    
32. 
    
33. for target in targets:
    
34.     name = str(target.text)+ ".txt"
    
35.     if not os.path.exists(path  + name):
    
36.         url = "https://www.bifeige.com" + str(target.a.get("href"))
    
37.         respound = openurl(url)
    
38.         save_text(respound, name)
    
39.         if i % 10 == 0:
    
40.             time.sleep(10)
    
41.         i += 1

复制代码

龙舞九天