爬虫

君子好逑

1. import urllib.request
   
2. import re
   
3. 
   
4. 
   
5. f = open('无敌真寂寞.txt','w')
   
6. 
   
7. 
   
8. def get_html(url):
   
9. 
   
10.     headers = {
    
11.     'Accept-Language': 'zh-CN',
    
12.     'Cache-Control': 'no-cache',
    
13.     'Connection': 'Keep-Alive',
    
14.     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363'
    
15.     }
    
16. 
    
17.     res = urllib.request.Request(url=url,headers=headers)
    
18. 
    
19.     response = urllib.request.urlopen(res)
    
20.     html = response.read().decode("gbk",'ignore')
    
21.     return html
    
22. 
    
23. def get_text(html):
    
24. 
    
25.     regular = re.compile(".*?<br><br>&nbsp;&nbsp;&nbsp;&nbsp;")
    
26. 
    
27.     m = regular.findall(html)
    
28. 
    
29.     for each in m:
    
30.         each = each[:-32]
    
31.         print('     ',end='')
    
32.         if '&nbsp;&nbsp;&nbsp;&nbsp;' in each :
    
33.             print(each.split(';')[-1])
    
34.             f.write(each.split(';')[-1])
    
35.             f.write('\n')
    
36.         else:
    
37.             print(each)
    
38.             f.write(each)
    
39.             f.write('\n')
    
40. 
    
41. def get_title(html):
    
42.     title = re.findall('<h1>(.*?)</h1>',html)
    
43.     title = title[0]
    
44.     return title
    
45. 
    
46. def get_next(html):
    
47.     url_1 = 'https://www.biduo.cc'
    
48.     regular = re.compile(r'.*?">上一章</a> &larr; <a href=".*?">章节列表</a> .*?; <a href="(.*?)"')
    
49.     m = regular.findall(html)
    
50.     url_2 = m.pop()
    
51.     next_address = url_1 + url_2
    
52.     return next_address
    
53. 
    
54. 
    
55. def main():
    
56.     number = int(input('请输入您想获取的章节总数：'))
    
57.     url = 'https://www.biduo.cc/biquge/39_39888/c13353637.html'
    
58.     for each in range(number):
    
59.         html = get_html(url)
    
60.         title = get_title(html)
    
61.         print(title)
    
62.         next_address = get_next(html)
    
63.         f.write(title)
    
64.         f.write('\n')
    
65.         f.write('\n')
    
66.         get_text(html)
    
67.         url = next_address
    
68.         f.write('\n')
    
69.         f.write('\n')
    
70. 
    
71.     f.close()
    
72. 
    
73. if __name__ == "__main__":
    
74.     main()

复制代码

爬的小说叫无敌真寂寞，喜欢看小说的可以试一试这个程序。有大佬要是把程序改进了能给我发一波吗，让我观摩观摩

小甲鱼的铁粉

1q23w31

1. import urllib.request
   
2. import re
   
3. 
   
4. 
   
5. f = open('无敌真寂寞.txt','w')
   
6. 
   
7. 
   
8. def get_html(url):
   
9. 
   
10.     headers = {
    
11.     'Accept-Language': 'zh-CN',
    
12.     'Cache-Control': 'no-cache',
    
13.     'Connection': 'Keep-Alive',
    
14.     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363'
    
15.     }
    
16. 
    
17.     res = urllib.request.Request(url=url,headers=headers)
    
18. 
    
19.     response = urllib.request.urlopen(res)
    
20.     html = response.read().decode("gbk",'ignore')
    
21.     return html
    
22. 
    
23. def get_text(html):
    
24. 
    
25.     regular = re.compile(".*?<br><br>&nbsp;&nbsp;&nbsp;&nbsp;")
    
26. 
    
27.     m = regular.findall(html)
    
28. 
    
29.     for each in m:
    
30.         each = each[:-32]
    
31.         if '&nbsp;&nbsp;&nbsp;&nbsp;' in each :
    
32.             f.write(each.split(';')[-1])
    
33.             f.write('\n')
    
34.         else:
    
35.             f.write(each)
    
36.             f.write('\n')
    
37. 
    
38. def get_title(html):
    
39.     title = re.findall('<h1>(.*?)</h1>',html)
    
40.     title = title[0]
    
41.     return title
    
42. 
    
43. def get_next(html):
    
44.     url_1 = 'https://www.biduo.cc'
    
45.     regular = re.compile(r'.*?">上一章</a> &larr; <a href=".*?">章节列表</a> .*?; <a href="(.*?)"')
    
46.     m = regular.findall(html)
    
47.     url_2 = m.pop()
    
48.     next_address = url_1 + url_2
    
49.     return next_address
    
50. 
    
51. 
    
52. def main():
    
53.     number = int(input('请输入您想获取的章节总数：'))
    
54.     url = 'https://www.biduo.cc/biquge/39_39888/c13353637.html'
    
55.     for each in range(number):
    
56.         html = get_html(url)
    
57.         title = get_title(html)
    
58.         next_address = get_next(html)
    
59.         f.write(title)
    
60.         f.write('\n'*2)
    
61.         get_text(html)
    
62.         url = next_address
    
63.         f.write('\n'*2)
    
64.     f.close()
    
65.     print('已完成')
    
66. 
    
67. if __name__ == "__main__":
    
68.     main()
    

复制代码

运行显示精简