爬取图片（没有加密）

Cool_Breeze

1. #!/usr/bin/env python3
   
2. #coding=utf-8
   
3. 
   
4. from special_str_replace import special_str_replace
   
5. import urllib.request,urllib.error
   
6. from bs4 import BeautifulSoup as bfs
   
7. import threading
   
8. import os
   
9. 
   
10. def main():
    
11.     url = 'https://www.woyaogexing.com/touxiang/z/qlshouhui/'
    
12.     home = 'https://www.woyaogexing.com'
    
13.     html = gethtml(url)
    
14.     for page_nu in get_page_list(html):
    
15.         get_photo_url_list(gethtml(home + page_nu))
    
16. 
    
17. def gethtml(url):
    
18.     head = {
    
19.     'Accept-Language': 'zh-CN,zh;q=0.9',
    
20.     "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.30 Safari/537.36"
    
21.     }
    
22.    
    
23.     req = urllib.request.Request(url=url, headers=head)
    
24.     response = urllib.request.urlopen(req)
    
25.     html = bfs(response,'html.parser') #解析html
    
26.     # print(html)
    
27.    
    
28.     return html
    
29. 
    
30. def get_page_list(html):
    
31.     data = []
    
32.     subject = html.find('div', class_="pMain")
    
33.    
    
34.     for i in subject.find_all('a', class_="img"):
    
35.         data.append(i.attrs['href'])
    
36.    
    
37.     # print(data)
    
38.     return data
    
39.    
    
40. def get_photo_url_list(html):
    
41.     #<h1>....</h1>
    
42.     title = str(html.find('h1').string).replace(':','_')
    
43.     #替换字符串中的特殊字符为'_'，为了创建文件夹
    
44.     title = special_str_replace(title)
    
45.     if not os.path.exists('./' + title):
    
46.         os.mkdir(title)
    
47.     os.chdir(title)
    
48.     #ul class="artCont cl"
    
49.     filterurl = html.find('ul', class_="artCont cl")
    
50.     ph_url = []
    
51.     for attr in filterurl.find_all('a'):
    
52.         # print(attr.attrs)
    
53.         ph_url.append(attr['href'])
    
54.         
    
55.     thread_photo(ph_url)
    
56.     os.chdir('../') #返回文件夹
    
57. def thread_photo(url):
    
58.     thread = []
    
59.     count = 0
    
60.     for i in url:
    
61.         count += 1
    
62.         thread.append(threading.Thread(target=get_ptoto, args=(i,count)))
    
63.     for i in thread:
    
64.         i.start()
    
65.     for i in thread:
    
66.         i.join()
    
67. def get_ptoto(u, count):
    
68.     print(u, '===>', count, '.jpeg')
    
69.     urllib.request.urlretrieve(\
    
70.         'https:' + u,
    
71.         str(count) + '.jpeg')
    
72. if __name__ == '__main__':
    
73.     main()

复制代码

1. #!/usr/bin/env python3
   
2. #coding=utf-8
   
3. 
   
4. 
   
5. def special_str_replace(special):
   
6.     limitstr = r'\/:*?"<>|'
   
7.     test = list(special)
   
8.     for index in range(len(test)):
   
9.         if test[index] in limitstr:
   
10.             test[index] = '_'
    
11.             
    
12.     return ''.join(test)

复制代码

Mike_python小

不好看

Cool_Breeze

Mike_python小 发表于 2020-6-10 16:48
不好看

拿来练手的！图片都是一个表情，没有什么好看的！

Cool_Breeze

至今还不会自动登录网页，进行爬取。不知道有没有现成的教学

Cool_Breeze

1. #!/usr/bin/env python3
   
2. #coding=utf-8
   
3. 
   
4. from special_str_replace import special_str_replace
   
5. import urllib.request,urllib.error
   
6. from bs4 import BeautifulSoup as bfs
   
7. import threading
   
8. import os
   
9. 
   
10. def main(page):
    
11.     url = 'https://www.woyaogexing.com/touxiang/z/qlshouhui/' + page
    
12.     home = 'https://www.woyaogexing.com'
    
13.     html = gethtml(url)
    
14.     for page_nu in get_page_list(html):
    
15.         get_photo_url_list(gethtml(home + page_nu))
    
16. 
    
17. def gethtml(url):
    
18.     head = {
    
19.     'Accept-Language': 'zh-CN,zh;q=0.9',
    
20.     "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.30 Safari/537.36"
    
21.     }
    
22.    
    
23.     req = urllib.request.Request(url=url, headers=head)
    
24.     response = urllib.request.urlopen(req)
    
25.     html = bfs(response,'html.parser') #解析html
    
26.     # print(html)
    
27.    
    
28.     return html
    
29. 
    
30. def get_page_list(html):
    
31.     data = []
    
32.     subject = html.find('div', class_="pMain")
    
33.    
    
34.     for i in subject.find_all('a', class_="img"):
    
35.         data.append(i.attrs['href'])
    
36.    
    
37.     # print(data)
    
38.     return data
    
39.    
    
40. def get_photo_url_list(html):
    
41.     #<h1>....</h1>
    
42.     title = str(html.find('h1').string).replace(':','_')
    
43.     #替换字符串中的特殊字符为'_'，为了创建文件夹
    
44.     title = special_str_replace(title)
    
45.     if not os.path.exists('./' + title):
    
46.         os.mkdir(title)
    
47.     os.chdir(title)
    
48.     #ul class="artCont cl"
    
49.     filterurl = html.find('ul', class_="artCont cl")
    
50.     ph_url = []
    
51.     for attr in filterurl.find_all('a'):
    
52.         # print(attr.attrs)
    
53.         ph_url.append(attr['href'])
    
54.         
    
55.     thread_photo(ph_url)
    
56.     os.chdir('../') #返回文件夹
    
57. def thread_photo(url):
    
58.     thread = []
    
59.     count = 0
    
60.     for i in url:
    
61.         count += 1
    
62.         thread.append(threading.Thread(target=get_ptoto, args=(i,count)))
    
63.     for i in thread:
    
64.         i.start()
    
65.     for i in thread:
    
66.         i.join()
    
67. def get_ptoto(u, count):
    
68.     print(u, '===>', count, '.jpeg')
    
69.     urllib.request.urlretrieve(\
    
70.         'https:' + u,
    
71.         str(count) + '.jpeg')
    
72. if __name__ == '__main__':
    
73.     for i in range(2,9):
    
74.         main('index_' + str(i) +'.html')

复制代码

形单影只的鱼

如何获取一个网页的所有图片呢  

鸬鹚鸟

Cool_Breeze 发表于 2020-6-12 19:12

666