零基础学习Python第56：OOXX

浮云骑士

import urllib.request
import os
import random


def url_open(url):
    req = urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.154 Safari/537.36 LBBROWSER')

    proxies = ['36.7.172.18:82','42.96.195.147:82','106.75.128.89:80']
    proxy = random.choice(proxies)

    proxy_support = urllib.request.ProxyHandler({'http':proxy})
    opener = urllib.request.build_opener(proxy_support)
    urllib.request.install_opener(opener)


    response = urllib.request.urlopen(url)
    html = response.read()

    return html

def get_page(url):
    html = url_open(url).decode('utf-8')

    a = html.find('current-comment-page')
    b = html.find(']',a)

    return html[a:b]


def find_imgs(url):
    html = url_open(url).decode('utf-8')
    img_addrs = []

    a = html.find('img src=')

    while a != -1:

        b = html.find('.jpg',a,a+255)
        if b!= -1:
            img_addrs.append(html[a+9:b+4])
        else:
            b = a+9

        a = html.find('img src=',b)

    return img_addrs




def save_imgs(folder,img_addrs):
    for each in img_addrs:
        filename = each.spilt('/')[-1]
        with open(filename, 'wb') as f:
            img = url_open(each)
            f.write(img)

def download_mm(folder = 'OOXX',pages = 10):
    os.mkdir(folder)
    os.chdir(folder)

    url = "http://jandan.net/ooxx/"
    page_num = int(get_page(url))

    for i in range(pages):
        page_num -= i
        page_url = url + 'page-' + str(page_num) + '#comments'
        img_addrs = find_imgs(page_url)
        save_imgs(folder,img_addrs)

if __name__ == '__main__':
    download_mm()
出现错误：
Traceback (most recent call last):
  File "C:\Users\lihongfei\Desktop\download_mm.py", line 74, in <module>
    download_mm()
  File "C:\Users\lihongfei\Desktop\download_mm.py", line 65, in download_mm
    page_num = int(get_page(url))
  File "C:\Users\lihongfei\Desktop\download_mm.py", line 24, in get_page
    html = url_open(url).decode('utf-8')
  File "C:\Users\lihongfei\Desktop\download_mm.py", line 18, in url_open
    response = urllib.request.urlopen(url)
  File "F:\python\lib\urllib\request.py", line 162, in urlopen
    return opener.open(url, data, timeout)
  File "F:\python\lib\urllib\request.py", line 471, in open
    response = meth(req, response)
  File "F:\python\lib\urllib\request.py", line 581, in http_response
    'http', request, response, code, msg, hdrs)
  File "F:\python\lib\urllib\request.py", line 503, in error
    result = self._call_chain(*args)
  File "F:\python\lib\urllib\request.py", line 443, in _call_chain
    result = func(*args)
  File "F:\python\lib\urllib\request.py", line 686, in http_error_302
    return self.parent.open(new, timeout=req.timeout)
  File "F:\python\lib\urllib\request.py", line 471, in open
    response = meth(req, response)
  File "F:\python\lib\urllib\request.py", line 581, in http_response
    'http', request, response, code, msg, hdrs)
  File "F:\python\lib\urllib\request.py", line 509, in error
    return self._call_chain(*args)
  File "F:\python\lib\urllib\request.py", line 443, in _call_chain
    result = func(*args)
  File "F:\python\lib\urllib\request.py", line 589, in http_error_default
    raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 503: Service Temporarily Unavailable

怎么办？

SixPy

善用【论坛搜索】功能，那里可能有您想要的答案！

论坛里讨论n遍了~~~~~~~~

浮云骑士

SixPy 发表于 2016-7-22 12:02
善用【论坛搜索】功能，那里可能有您想要的答案！

论坛里讨论n遍了~~~~~~~~

是吗？我看看

浮云骑士

SixPy 发表于 2016-7-22 12:02
善用【论坛搜索】功能，那里可能有您想要的答案！

论坛里讨论n遍了~~~~~~~~

好吧，好像是啊，我没注意。

浮云骑士

SixPy 发表于 2016-7-22 12:02
善用【论坛搜索】功能，那里可能有您想要的答案！

论坛里讨论n遍了~~~~~~~~

找到答案了。

frezs

浮云骑士 发表于 2016-7-22 14:48
找到答案了。

我也遇到相同情况，能帮忙解决一下吗？

浮云骑士

frezs 发表于 2016-7-26 00:03
我也遇到相同情况，能帮忙解决一下吗？

他们说简单网煎蛋网加了反爬虫机制，这是我在帖子里找到的能用的代码：
希望对你有用：

1. 
   
2. import urllib.request
   
3. import urllib.error
   
4. import os
   
5. import sys
   
6. import http.server
   
7. import http.client
   
8. import time
   
9. import re
   
10. import random
    
11. import math
    
12. 
    
13. data = None
    
14. headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36'}
    
15. enctype = 'utf-8'
    
16. proxies = []
    
17. max_error_times = 5        #最多允许失败5次，否则放弃该图片下载
    
18. 
    
19. def create_localhost():
    
20.     number = int((math.sqrt(5)-1)/2) * len(proxies)
    
21.     for x in range(number):
    
22.         proxies.append(None)
    
23. 
    
24. def get_result(req_or_url,is_retrieve=False,filename = None):         #flag是否使用retrieve
    
25.     error_time = 0
    
26.     while True:
    
27.         try:
    
28.             if error_time == max_error_times:
    
29.                 print('失败次数达%d次......放弃操作' % max_error_times)
    
30.                 return None
    
31.             error_time += 1
    
32.             if is_retrieve:
    
33.                 return urllib.request.urlretrieve(req_or_url,filename)
    
34.             else:
    
35.                 return urllib.request.urlopen(req_or_url)
    
36.         except urllib.error.URLError as e:
    
37.             if hasattr(e,'code'):         
    
38.                 print(e.code,e.reason)
    
39.                 change_proxy()
    
40.                 continue
    
41.             elif hasattr(e,'reason'):
    
42.                 print(e)
    
43.                 change_proxy()
    
44.                 continue
    
45.         except (ConnectionResetError,http.client.BadStatusLine) as e:
    
46.             print(e)
    
47.             change_proxy()
    
48.             continue
    
49.         except TimeoutError as e:
    
50.             print(e)
    
51.             print('服务器长时间无响应，自动切换代理.....')
    
52.             change_proxy()
    
53.             continue
    
54. 
    
55. def get_proxy():
    
56.     global data,headers,proxies
    
57.     req = urllib.request.Request('http://www.xici.net.co',None,headers)
    
58.     response = get_result(req)
    
59.     html = response.read().decode('utf-8')
    
60.     p = re.compile(r'''<tr\sclass[^>]*>\s+
    
61.                                     <td>.+</td>\s+
    
62.                                     <td>(.*)?</td>\s+
    
63.                                     <td>(.*)?</td>\s+
    
64.                                     <td>(.*)?</td>\s+
    
65.                                     <td>(.*)?</td>\s+
    
66.                                     <td>(.*)?</td>\s+
    
67.                                     <td>(.*)?</td>\s+
    
68.                                 </tr>''',re.VERBOSE)
    
69.     proxy_list = p.findall(html)
    
70.     for each_proxy in proxy_list[1:]:
    
71.         if each_proxy[4] == 'HTTP':
    
72.             proxies.append(each_proxy[0]+':'+each_proxy[1])
    
73. 
    
74. def change_proxy():
    
75.     proxy = random.choice(proxies)
    
76.     if proxy == None:
    
77.         proxy_support = proxy_support = urllib.request.ProxyHandler({})
    
78.     else:
    
79.         proxy_support = urllib.request.ProxyHandler({'http':proxy})
    
80.     opener = urllib.request.build_opener(proxy_support)
    
81.     opener.addheaders = [('User-Agent',headers['User-Agent'])]
    
82.     urllib.request.install_opener(opener)
    
83.     print('智能切换代理：%s' % ('本机' if proxy==None else proxy))
    
84. 
    
85. def get_page():         #获取最大页数
    
86.     home = 'http://jandan.net/ooxx'
    
87.     global data,headers,enctype
    
88.     req = urllib.request.Request(home,data,headers)
    
89.     response = get_result(req)
    
90.     html = response.read().decode(enctype)
    
91.     find_string = 'current-comment-page'
    
92.     find_start = html.index(find_string) + len(find_string) + 3
    
93.     find_end = html.index(']',find_start+1)
    
94.     return int(html[find_start:find_end])
    
95. test = None
    
96. def get_pic(page):      #生成器，返回一个图片链接
    
97.     global data,headers,enctype
    
98.     while True:
    
99.         url = 'http://jandan.net/ooxx/page-%d' % page
    
100.         print('当前页面：%d' % page)
     
101.         req = urllib.request.Request(url,data,headers)
     
102.         response = get_result(req)
     
103.         if response == None:
     
104.             print('获取页面失败.....')
     
105.             sys.exit()
     
106.         html = response.read().decode(enctype)
     
107.         pic = re.compile(r'<img\s+src="(http://.+?\.(?:jpg|jpeg|gif))"')
     
108.         for pic in pic.finditer(html):
     
109.             yield pic.group(1)
     
110.         time.sleep(5)
     
111.         page -= 1
     
112.         if page<1:
     
113.             break
     
114. 
     
115. save_path = 'G:\\图片\\妹子图'
     
116. 
     
117. def download():
     
118.     count = 1
     
119.     global data,headers
     
120.     for pic_url in get_pic(get_page()):         #get_page()改为页数如1000可从1000页开始下载
     
121.         file_name = os.path.split(pic_url)[1]
     
122.         if not os.path.isdir(save_path):    #目录不存在就创建
     
123.             os.makedirs(save_path)
     
124.         get_result(pic_url,True,save_path+'\\'+file_name)
     
125.         print('本次成功下载第%d个图片! %s' % (count , pic_url))
     
126.         count += 1
     
127. 
     
128. if __name__ == '__main__':
     
129.     get_proxy()
     
130.     create_localhost()
     
131.     download()
     

复制代码

那是个玩笑话

............

那是个玩笑话

.............................df

那是个玩笑话