爬虫批量下载图片时，不知道哪里出问题，百思不得其解，求高人指点。。。。,Python交流,技术交流,鱼C论坛

8wy403208 发表于 2021-7-7 14:42:18

爬虫批量下载图片时，不知道哪里出问题，百思不得其解，求高人指点。。。。

import random
import requests
import requests
from lxml import etree
import time
import threading
from fake_useragent import UserAgent
import re

ua = ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"]

a = random.choice(ua)
#弄了好久都弄不出来随机UA O_O

iplist = ['http://47.104.96.77:80', 'http://150.255.144.238:9999', 'http://58.253.158.205:9999', 'http://223.243.172.212:9999', 'http://193.149.225.242:80', 'http://27.191.60.19:3256', 'http://59.55.160.111:3256', 'http://42.176.134.82:9999', 'http://223.214.219.26:3256', 'http://123.171.42.233:3256', 'http://36.56.102.202:9999', 'http://175.146.211.63:9999', 'http://175.174.190.195:9999', 'http://59.63.75.2:9999', 'http://36.56.101.187:9999', 'http://8.208.91.118:8000', 'http://42.57.89.33:9999', 'http://60.174.188.167:9999', 'http://60.174.188.185:9999', 'http://58.255.207.137:9999', 'http://60.7.97.178:9999', 'http://59.33.54.242:9999', 'http://27.43.191.71:9999', 'http://60.174.188.109:9999', 'http://220.249.149.101:9999', 'http://60.5.41.230:9999', 'http://27.43.189.182:9999', 'http://58.255.206.248:9999', 'http://139.224.211.179:8080', 'http://118.99.127.27:8080', 'http://60.167.21.108:1133', 'http://60.168.207.144:1133', 'http://191.101.39.254:80', 'http://27.43.188.50:9999', 'http://110.18.154.209:9999', 'http://27.40.93.213:9999', 'http://104.2.97.167:80', 'http://27.43.186.91:9999', 'http://114.99.3.151:8888', 'http://183.166.171.232:8888', 'http://210.153.218.246:8080', 'http://175.44.109.140:9999', 'http://58.253.152.253:9999', 'http://45.70.15.4:8080', 'http://182.105.200.207:9000', 'http://220.201.84.117:9999', 'http://163.204.95.30:9999', 'http://27.40.90.208:9999', 'http://58.253.155.138:9999', 'http://80.79.66.82:3629', 'http://27.43.185.162:9999', 'http://87.103.173.158:8080', 'http://163.204.92.75:9999', 'http://218.91.7.126:9999', 'http://45.231.215.247:999', 'http://85.29.147.222:4145', 'http://58.253.148.36:9999', 'http://36.248.133.210:9999', 'http://58.253.159.101:9999', 'http://117.94.142.45:9999', 'http://42.7.29.5:9999', 'http://36.250.156.85:9999', 'http://58.253.157.131:9999', 'http://59.63.75.159:9999', 'http://181.3.146.133:10809', 'http://60.17.207.230:9999', 'http://27.43.186.250:9999', 'http://222.141.249.132:9999', 'http://113.237.7.86:9999', 'http://109.248.249.29:8081', 'http://120.83.111.140:9999', 'http://123.200.20.242:58847', 'http://104.129.198.162:10605', 'http://27.40.126.132:9999', 'http://54.237.31.235:80', 'http://116.68.162.105:1080', 'http://163.179.205.7:9999', 'http://175.174.190.177:9999', 'http://49.86.9.8:9999', 'http://59.55.164.19:3256', 'http://36.250.156.153:9999', 'http://113.58.176.214:9999', 'http://180.122.141.121:9999', 'http://124.94.248.243:9999', 'http://59.33.53.224:9999', 'http://47.100.207.255:8080', 'http://185.204.187.53:5075', 'http://104.129.194.175:8800', 'http://183.166.87.191:8888', 'http://104.129.192.76:8800', 'http://120.86.38.148:9999', 'http://36.248.132.198:9999', 'http://163.204.93.129:9999', 'http://182.46.98.51:9999', 'http://223.247.168.42:9999', 'http://200.6.169.251:31246', 'http://113.237.5.209:9999', 'http://82.156.171.147:3128', 'http://36.56.103.172:9999', 'http://58.253.157.103:9999', 'http://213.230.127.139:3128', 'http://60.174.189.198:9999', 'http://60.17.204.74:9999', 'http://106.45.105.253:3256', 'http://117.69.201.100:8888', 'http://27.214.112.204:9000', 'http://123.169.98.239:9999', 'http://60.7.210.161:9999', 'http://42.238.70.36:9999', 'http://60.5.172.61:9999', 'http://89.22.17.62:61616', 'http://103.81.114.140:3128', 'http://191.97.18.161:999', 'http://171.35.172.185:9999', 'http://117.94.142.26:9999', 'http://36.249.49.39:9999', 'http://89.33.192.33:3128', 'http://60.174.188.21:9999', 'http://60.19.238.165:9999', 'http://124.94.250.229:9999', 'http://175.146.209.202:9999', 'http://223.240.208.74:1133', 'http://115.61.42.63:9999', 'http://59.33.57.184:9999', 'http://36.56.102.60:9999', 'http://27.152.194.220:8888', 'http://191.101.39.28:80', 'http://36.56.100.151:9999', 'http://106.45.220.29:3256', 'http://27.152.193.159:8888', 'http://49.86.56.86:9999', 'http://223.215.119.200:9999', 'http://58.17.103.3:9999', 'http://60.169.134.204:9999', 'http://117.69.28.40:9999', 'http://1.70.64.46:9999', 'http://27.43.186.13:9999', 'http://140.255.188.118:9999', 'http://202.72.209.3:44550', 'http://1.70.67.16:9999', 'http://60.195.62.81:80', 'http://191.101.39.59:80', 'http://182.87.129.240:9999', 'http://113.229.4.38:9999', 'http://177.126.216.149:60375', 'http://58.97.208.8:8080', 'http://1.70.65.127:9999', 'http://121.19.6.49:9999', 'http://113.229.2.60:9999', 'http://81.163.36.210:42967', 'http://220.201.86.225:9999', 'http://113.194.138.160:9999', 'http://114.231.45.107:8888', 'http://171.35.146.173:9999', 'http://117.94.143.36:9999', 'http://96.9.77.71:8080', 'http://192.109.165.166:80', 'http://1.70.65.42:9999', 'http://191.101.39.81:80', 'http://1.70.65.49:9999', 'http://115.219.1.155:3256', 'http://59.14.228.30:80', 'http://42.7.29.143:9999', 'http://58.255.206.45:9999', 'http://1.70.64.201:9999', 'http://45.122.57.226:8085', 'http://180.211.248.222:8080', 'http://171.35.140.105:9999', 'http://114.100.0.200:9999', 'http://113.194.139.79:9999', 'http://103.80.237.211:3888', 'http://175.146.211.10:9999', 'http://58.253.152.163:9999', 'http://175.212.226.1:80', 'http://115.218.2.121:9000', 'http://36.56.102.72:9999', 'http://220.189.70.111:8888', 'http://121.230.210.200:3256', 'http://27.43.187.48:9999', 'http://175.44.109.15:9999', 'http://168.195.196.25:53496', 'http://117.69.43.195:9999', 'http://114.100.0.178:9999', 'http://113.237.4.48:9999', 'http://218.87.174.10:9999', 'http://59.33.56.109:9999', 'http://121.232.194.228:9000', 'http://65.0.27.148:3128', 'http://117.35.255.91:3000', 'http://60.167.102.61:1133', 'http://27.43.184.133:9999', 'http://23.97.77.195:8118', 'http://220.249.149.147:9999', 'http://106.45.104.199:3256', 'http://36.56.100.27:9999', 'http://104.129.204.84:8800', 'http://164.90.179.64:8118', 'http://124.205.153.59:80', 'http://110.18.152.90:9999', 'http://114.239.210.200:9999', 'http://115.62.42.14:9999', 'http://223.243.228.127:9999', 'http://49.86.9.32:9999', 'http://113.237.3.49:9999', 'http://115.77.191.180:53281', 'http://59.33.56.35:9999', 'http://113.194.29.13:9999', 'http://36.250.156.63:9999', 'http://117.69.43.235:9999', 'http://182.84.145.128:3256', 'http://121.232.148.243:3256', 'http://163.204.94.56:9999', 'http://223.240.208.161:8888', 'http://121.232.148.47:9000', 'http://112.195.242.33:3256', 'http://60.174.191.130:9999', 'http://121.230.211.253:3256', 'http://183.165.129.194:8888', 'http://43.239.152.126:8080', 'http://1.70.67.221:9999', 'http://202.59.137.98:8080', 'http://113.237.1.198:9999', 'http://103.52.211.174:80', 'http://222.112.240.139:80', 'http://42.7.30.118:9999', 'http://182.46.102.57:9999', 'http://72.169.66.137:87', 'http://54.220.139.200:80', 'http://114.233.2.49:9999', 'http://120.11.3.56:9999', 'http://59.33.58.76:9999', 'http://171.35.209.99:9999', 'http://117.64.237.220:9999', 'http://36.248.132.27:9999', 'http://103.115.14.37:80', 'http://118.212.106.149:9999', 'http://119.114.16.45:9999', 'http://113.194.49.177:9999', 'http://114.97.184.52:8888', 'http://115.61.40.15:9999', 'http://113.194.50.155:9999', 'http://220.201.84.252:9999', 'http://27.152.194.56:8888', 'http://60.174.190.198:9999', 'http://210.113.122.29:80', 'http://45.179.200.129:999', 'http://45.125.63.46:44110', 'http://182.253.6.3:8080', 'http://113.195.170.192:9999', 'http://96.9.69.164:53281', 'http://42.56.239.15:9999', 'http://125.87.95.217:3256', 'http://113.194.129.232:9999', 'http://183.166.7.56:8888', 'http://160.16.77.108:3128', 'http://27.43.186.84:9999', 'http://107.150.37.85:3128', 'http://60.177.157.131:9000', 'http://103.122.67.190:3127', 'http://115.219.1.167:3256', 'http://60.168.206.168:8888', 'http://186.215.96.10:8080', 'http://104.254.238.122:38361', 'http://27.159.167.47:8888', 'http://149.5.37.1:8080', 'http://121.230.117.27:9999', 'http://42.238.65.25:9999', 'http://117.94.116.48:9000', 'http://165.227.173.87:39399', 'http://103.224.39.2:83', 'http://193.149.225.64:80', 'http://220.135.2.247:59171', 'http://124.94.252.137:9999', 'http://120.83.105.26:9999', 'http://182.87.138.129:9999', 'http://210.246.240.42:14153', 'http://113.194.137.183:9999', 'http://36.56.103.48:9999', 'http://117.69.43.234:9999', 'http://192.109.165.203:80', 'http://106.45.104.88:3256', 'http://175.42.68.203:9999', 'http://113.229.0.173:9999', 'http://103.26.54.94:8080', 'http://58.255.6.142:9999', 'http://201.249.161.51:999', 'http://114.100.2.111:9999', 'http://113.195.153.62:9999', 'http://165.16.96.92:8080', 'http://193.149.225.101:80', 'http://171.35.211.186:9999', 'http://103.124.136.73:8080', 'http://113.194.130.84:9999', 'http://47.90.132.228:81', 'http://131.153.151.250:20367', 'http://103.15.140.177:44759', 'http://1.70.65.191:9999', 'http://103.99.8.106:84', 'http://158.140.167.148:53281', 'http://175.42.122.124:9999', 'http://114.233.2.152:9999', 'http://102.134.123.167:8080', 'http://223.243.177.159:9999', 'http://114.100.3.46:9999', 'http://61.147.165.195:888', 'http://106.45.104.198:3256', 'http://117.69.230.4:3256', 'http://177.242.130.194:8080', 'http://104.129.196.81:8800', 'http://203.143.33.249:8080', 'http://94.23.173.11:43567', 'http://180.119.92.135:9999', 'http://163.179.199.161:9999', 'http://111.77.113.59:3256', 'http://112.91.75.220:9999', 'http://59.33.58.70:9999', 'http://177.73.250.160:33172', 'http://59.33.53.67:9999', 'http://187.44.230.83:3128', 'http://175.7.199.2:3256', 'http://106.45.104.109:3256', 'http://110.18.153.91:9999', 'http://111.118.128.140:41344', 'http://113.120.62.164:9999', 'http://103.110.184.109:8181', 'http://105.112.1.166:8080', 'http://180.122.75.180:9999', 'http://112.195.240.201:3256', 'http://103.153.232.3:8080', 'http://41.90.245.23:8080', 'http://41.33.66.248:34560', 'http://60.174.188.129:9999', 'http://49.88.63.206:8000', 'http://59.33.99.100:9999', 'http://36.251.144.41:9999', 'http://182.87.138.228:3256', 'http://60.174.190.206:9999', 'http://187.58.98.145:8080', 'http://36.66.16.193:8080', 'http://27.40.93.59:9999', 'http://49.86.59.113:9999', 'http://218.73.133.219:8888', 'http://36.251.148.60:9999', 'http://104.129.202.120:8800', 'http://27.43.190.120:9999', 'http://101.74.83.173:9999', 'http://42.56.239.110:9999', 'http://113.121.93.53:9999', 'http://14.38.255.42:80', 'http://59.63.75.141:9999', 'http://182.253.25.211:8080', 'http://183.165.128.237:8888', 'http://223.247.169.106:9999', 'http://1.196.105.132:9999', 'http://47.100.207.132:8080', 'http://165.22.60.129:33971', 'http://59.33.58.89:9999', 'http://27.40.126.100:9999', 'http://27.40.126.66:9999', 'http://113.195.234.136:9999', 'http://213.6.28.91:8080', 'http://171.35.213.163:9999', 'http://189.85.102.1:999', 'http://103.99.8.102:83', 'http://163.179.199.195:9999', 'http://49.71.142.185:9999', 'http://36.249.48.29:9999', 'http://138.97.200.73:8080', 'http://113.237.1.237:9999', 'http://182.87.140.68:9999', 'http://45.186.60.6:8085', 'http://27.40.90.29:9999', 'http://175.44.109.132:9999', 'http://182.87.150.17:9999', 'http://45.32.49.225:3128', 'http://185.93.71.159:33128', 'http://60.5.172.125:9999', 'http://180.122.141.222:9999', 'http://47.242.77.237:8099', 'http://171.35.141.35:9999', 'http://60.195.172.79:80', 'http://112.195.243.42:3256', 'http://78.140.6.68:53281', 'http://110.86.137.18:8888', 'http://60.7.163.19:9999', 'http://58.253.159.99:9999', 'http://183.166.66.106:9999', 'http://120.86.39.69:9999', 'http://36.56.102.101:9999', 'http://103.146.30.178:8080', 'http://223.243.244.190:9999', 'http://60.168.207.244:1133', 'http://49.86.183.209:9999', 'http://45.228.77.242:999', 'http://42.176.135.193:9999', 'http://36.56.101.147:9999', 'http://120.6.216.64:9999', 'http://42.7.28.46:9999', 'http://103.99.176.32:8080', 'http://175.42.128.203:9999', 'http://138.97.200.84:8080', 'http://121.8.215.106:9797', 'http://104.129.204.32:8800', 'http://115.219.2.112:3256', 'http://223.243.48.109:9999', 'http://49.86.179.122:9999', 'http://218.64.142.253:9999', 'http://154.127.36.138:8080', 'http://154.120.192.6:8080', 'http://103.111.55.210:47623', 'http://60.174.189.6:9999', 'http://42.177.139.67:9999', 'http://41.77.81.90:8080', 'http://113.195.152.167:9999', 'http://113.195.235.70:9999', 'http://138.97.67.205:8080', 'http://42.7.7.43:9999', 'http://103.213.213.22:84', 'http://120.86.38.162:9999', 'http://58.253.147.102:9999', 'http://150.255.150.238:9999', 'http://116.197.128.197:8080', 'http://27.131.179.220:10443', 'http://1.70.67.247:9999', 'http://89.111.105.95:41258', 'http://175.42.123.66:9999', 'http://60.2.44.182:52758', 'http://190.107.20.93:999', 'http://27.40.125.241:9999', 'http://113.237.4.68:9999', 'http://59.63.75.82:9999', 'http://185.76.215.54:80', 'http://60.174.188.247:9999', 'http://118.212.107.151:9999', 'http://36.249.109.17:9999', 'http://163.179.206.128:9999', 'http://117.69.63.113:9999', 'http://175.42.128.93:9999', 'http://175.146.211.168:9999', 'http://138.185.72.41:8080', 'http://112.195.243.90:3256', 'http://45.167.89.91:999', 'http://131.153.151.250:37701', 'http://58.255.7.128:9999', 'http://121.230.210.128:3256', 'http://112.195.243.181:3256', 'http://221.199.195.49:9999', 'http://27.159.165.146:8888', 'http://181.225.41.131:8080', 'http://177.11.84.200:8080', 'http://213.14.32.75:9090', 'http://217.79.189.28:3128', 'http://125.40.211.51:9999', 'http://117.94.140.249:9999', 'http://163.179.206.39:9999', 'http://60.174.188.71:9999', 'http://45.160.7.78:8081', 'http://45.251.228.217:8080', 'http://191.209.54.111:8080', 'http://182.253.82.154:37242', 'http://58.21.183.136:9999', 'http://27.40.106.39:9999', 'http://45.242.91.46:8080', 'http://49.70.99.35:9999', 'http://110.229.152.82:9999', 'http://186.219.34.210:4145', 'http://175.7.199.33:3256', 'http://125.141.117.44:80', 'http://42.238.71.205:9999', 'http://115.218.5.183:9000', 'http://182.87.139.104:9999', 'http://175.43.59.62:9999', 'http://114.239.145.182:9999', 'http://142.93.220.78:3128', 'http://175.146.211.165:9999', 'http://47.100.245.127:8080', 'http://175.101.80.138:8080', 'http://106.45.104.139:3256', 'http://117.64.236.120:1133', 'http://14.126.15.230:9999', 'http://58.253.158.252:9999', 'http://42.177.137.68:9999', 'http://220.189.70.208:8888', 'http://58.255.4.120:9999', 'http://27.43.188.151:9999', 'http://27.43.188.218:9999', 'http://216.254.142.47:80', 'http://165.16.46.215:8080', 'http://101.18.81.240:9999', 'http://140.255.139.90:3256', 'http://175.146.215.215:9999', 'http://182.87.136.186:9999', 'http://124.94.252.223:9999', 'http://60.19.236.54:9999', 'http://42.59.103.207:9999', 'http://36.89.8.235:8080', 'http://112.195.242.223:3256', 'http://175.42.129.98:9999', 'http://59.55.162.207:3256', 'http://141.98.112.13:8080']
ip = random.choice(iplist)

#IP池也弄不出来，O_O ，

for i in range(2,52):

url = f'https://www.umei.net/bizhitupian/meinvbizhi/index_{i+10}.htm'
resp = requests.get(url,headers={'UserAgent':a},proxies={'http':ip},timeout=14)



print(resp.status_code,'这个OK')
resp.close()
time.sleep(3)
page = etree.HTML(resp.text)
ziye = page.xpath('/html/body/div/div/ul/li/a/@href')

#小图_url
for aa in ziye:

   yemian = 'https://www.umei.net' + aa

   ye = requests.get(yemian,headers={'UserAgent':a},proxies={'http':ip},timeout=14)
   ye.encoding = 'utf-8'
   print(ye.status_code,'这个也OK')
   #print(ye.text)
   ye.close()
   time.sleep(3)

   #大图_url
   zi_tu = etree.HTML(ye.text)
   zitu_url = zi_tu.xpath('/html/body/div/div/p/img/@src')
   tu_url = ''.join(zitu_url)
   print(tu_url)
   tulist.append(tu_url)
   # 名字

   lingyi_tit = zi_tu.xpath('/html/body/div/p/text()')
   ai = ''.join(lingyi_tit)
   mz = ''.join(re.findall(r'.*?组《(?P<dd>.*?)》图',ai)).replace(' ','').replace('“”','').replace('”','').replace('《','').replace('》','').replace('_','').replace('\\','').replace('/','')
   print(mz)

   #访问大图并保存

   tupian = requests.get(tu_url,headers={'UserAgent':a},proxies={'http':ip},timeout=9)


   with open('E:美女壁纸/%s.jpg'%mz,'wb') as f:

         f.write(tupian.content)
         print('<%s>下载好了'%mz)
         f.close()
         time.sleep(3)

print(url+' : 这页OK')

高手，这个怎么解决？。。。。。

qq1084799589 发表于 2021-7-7 15:04:28

User_Agent = UserAgent(use_cache_server=False)# 实例化UserAgent
headers = {
"user-agent": User_Agent.random
}

wx626810 发表于 2021-7-7 18:01:44

proxy的问题，如果是网上找的免费的，多半是不能用的

8wy403208 发表于 2021-7-9 22:11:40

wx626810 发表于 2021-7-7 18:01
proxy的问题，如果是网上找的免费的，多半是不能用的

是的，在网上随便找的代理IP，，，，，
有什么办法能解决这个问题吗？{:5_100:}

大马强 发表于 2021-7-10 16:20:48

8wy403208 发表于 2021-7-9 22:11
是的，在网上随便找的代理IP，，，，，
有什么办法能解决这个问题吗？

得花钱{:10_327:}

rowdoge 发表于 2021-7-12 14:31:24

免费代理太慢池超时了，不用代理试试看

页: [1]

鱼C论坛's Archiver

爬虫批量下载图片时，不知道哪里出问题，百思不得其解，求高人指点。。。。