多进程图片下载
本帖最后由 qq1084799589 于 2021-9-7 18:01 编辑# -*- coding: utf-8 -*-
import requests
from multiprocessing import Pool
import multiprocessing
import os
import time
from fake_useragent import UserAgent
User_Aget = UserAgent(use_cache_server=False)
# 创建保存图片文件夹
try:
os.makedirs(os.path.split(os.path.realpath(__file__)) + '\\图片\\' )
except:
pass
IMAGE_PATH = os.path.split(os.path.realpath(__file__)) + '\\图片\\'
n = 1
info_lists=[]
csv_str = open('ftp.csv', 'r')
# 按行读取
list_d = csv_str.readlines()
jj = []
for d in list_d:
str_list = d.strip('\n|\"')
if len(str_list) != 0:
jj.append(str_list)
def get_info(url,n):
try:
headers = {
'User-Agent': User_Aget.random
}
photos_url_str = requests.get(url, headers=headers)
time.sleep(3)
pic = photos_url_str.content
# 图片内容需要以二进制wb读写
photo = open(IMAGE_PATH + str(n) + '.jpg', 'wb')
# 获取pic的二进制内容
photo.write(pic)
# 关闭文件
photo.close()
except requests.RequestException:
jj.append(url)
if __name__=='__main__':
start_1=time.time()
for url in jj:
get_info(url,n)
n += 1
#start_3=time.time()
pool = Pool(multiprocessing.cpu_count())
pool.map(get_info, jj)
#end_3 = time.time()
页:
[1]