多进程图片下载

qq1084799589

1. # -*- coding: utf-8 -*-
   
2. import requests
   
3. from multiprocessing import Pool
   
4. import multiprocessing
   
5. import os
   
6. import time
   
7. from fake_useragent import UserAgent
   
8. 
   
9. 
   
10. User_Aget = UserAgent(use_cache_server=False)
    
11. #   创建保存图片文件夹
    
12. try:
    
13.     os.makedirs(os.path.split(os.path.realpath(__file__))[0] + '\\图片\\' )
    
14. except:
    
15.     pass
    
16. IMAGE_PATH = os.path.split(os.path.realpath(__file__))[0] + '\\图片\\'
    
17. n = 1
    
18. info_lists=[]
    
19. csv_str = open('ftp.csv', 'r')
    
20. #   按行读取
    
21. list_d = csv_str.readlines()
    
22. jj = []
    
23. for d in list_d:
    
24.     str_list = d.strip('\n|"')
    
25.     if len(str_list) != 0:
    
26.         jj.append(str_list)
    
27. def get_info(url,n):
    
28.     try:
    
29.         headers = {
    
30.             'User-Agent': User_Aget.random
    
31.         }
    
32. 
    
33.         photos_url_str = requests.get(url, headers=headers)
    
34.         time.sleep(3)
    
35.         pic = photos_url_str.content
    
36.         # 图片内容需要以二进制wb读写
    
37. 
    
38.         photo = open(IMAGE_PATH + str(n) + '.jpg', 'wb')
    
39. 
    
40.         # 获取pic的二进制内容
    
41.         photo.write(pic)
    
42.         #    关闭文件
    
43.         photo.close()
    
44.     except requests.RequestException:
    
45.         jj.append(url)
    
46. 
    
47. 
    
48. 
    
49. if __name__=='__main__':
    
50. 
    
51.     start_1=time.time()
    
52.     for url in jj:
    
53.         get_info(url,n)
    
54.         n += 1
    
55.     #start_3=time.time()
    
56.     pool = Pool(multiprocessing.cpu_count())
    
57.     pool.map(get_info, jj)
    
58.     #end_3 = time.time()
    

复制代码