为什么requests的代理不能用，如果用urllib.request就行

幽梦三影

1. import requests
   
2. import urllib.request
   
3. import random
   
4. import re
   
5. import os
   
6. 
   
7. def install():
   
8.     users = [
   
9.     "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0",
   
10.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; Maxthon/3.0)",
    
11.     "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ;  QIHU 360EE)"
    
12.     ]
    
13.     ips = [
    
14.     "http://61.191.41.130",
    
15.     "http://122.238.12.191",
    
16.     "http://222.161.56.166"
    
17.     ]
    
18.     ip = random.choice(ips)
    
19.     user = random.choice(users)
    
20.     return ip,user
    
21.    
    
22. def get_html(url):
    
23.     num = install()
    
24.     rep = requests.get(url,data={'User-Agent':num[1]},proxies={'https':num[0]},stream=True)
    
25.     return rep
    
26. 
    
27. name = input('请输入要查找的商品：')
    
28. num = int(input('请输入页数：'))
    
29. 
    
30. key = urllib.request.quote(name)
    
31. 
    
32. os.mkdir(name)
    
33. os.chdir(name)
    
34.    
    
35. def main():
    
36.     for j in range(num):
    
37.         url = 'https://s.taobao.com/search?q=' + key + '&s='+ str(j*44)
    
38.         html = get_html(url).text
    
39.         s = re.findall(r'"pic_url":"([^"]+?)".+?"view_price":"([^"]+?)".+?"view_sales":"([^"]+?)"',html)
    
40.         
    
41.         
    
42.         os.mkdir('%s%d'%(name,j+1))
    
43.         os.chdir('%s%d'%(name,j+1))
    
44.    
    
45.         for i in s:
    
46.             img = get_html('http:' + str(i[0]))#在这获取html不能使用代理
    
47.             print(img)
    
48.             
    
49.             with open('%s元_%s.jpg'%(i[1],i[2]),'wb') as f:
    
50.                 f.write(img.content)
    
51.    
    
52.         os.chdir(os.pardir)
    
53.         
    
54. if __name__ == '__main__':
    
55.     main()
    
56. 
    
57. 
    
58.   
    
59. 
    

复制代码

幽梦三影

iwanna 发表于 2018-7-10 09:29
可能是你ip是http，proxies里面写的是https
另外，user-angent是放在data里吗，我记得应该放在headers里{: ...

是headers，我弄错了

幽梦三影

iwanna 发表于 2018-7-10 09:29
可能是你ip是http，proxies里面写的是https
另外，user-angent是放在data里吗，我记得应该放在headers里{: ...

爬一张图还行，多张图就不行了

幽梦三影

iwanna 发表于 2018-7-10 09:58
不行是什么不行，报的什么错

raise ProxyError(e, request=request)
requests.exceptions.ProxyError: HTTPConnectionPool(host='222.161.56.166', port=80): Max retries exceeded with url: http://g-search1.alicdn.com/img/ ... !0-saturn_solar.jpg (Caused by ProxyError('Cannot connect to proxy.', NewConnectionError('<urllib3.connection.HTTPConnection object at 0x0000021D734E6B38>: Failed to establish a new connection: [WinError 10061] 由于目标计算机积极拒绝，无法连接。'))

幽梦三影

我没办法，只能这样了

1. import requests
   
2. import urllib.request as u
   
3. import random
   
4. import re
   
5. import os
   
6. 
   
7. def get_html(url):
   
8.     user = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0"
   
9.     ips = [
   
10.     '222.161.56.166',
    
11.     '122.238.12.191',
    
12.     '61.191.41.130'
    
13.         ]
    
14.     ip = random.choice(ips)
    
15.     p = u.ProxyHandler({'http':ip})
    
16.     opener = u.build_opener(p)
    
17.     u.install_opener(opener)
    
18.    
    
19.     rep = requests.get(url,headers={'User-Agent':user})
    
20.     return rep
    
21. 
    
22. name = input('请输入要查找的商品：')
    
23. num = int(input('请输入页数：'))
    
24. 
    
25. key = u.quote(name)
    
26. 
    
27. os.mkdir(name)
    
28. os.chdir(name)
    
29.    
    
30. def main():
    
31.     for j in range(num):
    
32.         url = 'https://s.taobao.com/search?q=' + key + '&s='+ str(j*44)
    
33.         html = get_html(url).text
    
34.         s = re.findall(r'"pic_url":"([^"]+?)".+?"view_price":"([^"]+?)".+?"view_sales":"([^"]+?)"',html)
    
35.         
    
36.         os.mkdir('%s%d'%(name,j+1))
    
37.         os.chdir('%s%d'%(name,j+1))
    
38.         
    
39.         for i in s:
    
40.             img = get_html(url=('http:' + i[0]))
    
41.             
    
42.             with open('%s元_%s.jpg'%(i[1],i[2]),'wb') as f:
    
43.                 f.write(img.content)
    
44.    
    
45.         os.chdir(os.pardir)
    
46.         
    
47. if __name__ == '__main__':
    
48.     main()
    
49. 
    
50. 
    
51.   
    
52. 
    

复制代码