56课爬虫代码报错,附代码求助

tengfei8382

问题:
可以爬第一页的全部6张图片,然后就报错urllib.error.URLError: <urlopen error unknown url type: "http>
百度了一下基本全是英文页面,尝试翻译还是不懂
版本: python 3.5.2
运行环境:win7 64位

指导一下问题出在哪?
        



import urllib.request
import os


def url_open(url):

    req = urllib.request.Request(url)
    req.add_header('user-agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0')
    html = urllib.request.urlopen(req).read()

    return html


def get_page_num(url):

    html = url_open(url).decode('utf-8')

    a = html.find('current-comment-page">[') + 23
    b = html.find(']', a)

    return html[a:b]

   
def find_img(url):

    html = url_open(url).decode('utf-8')

    img_addrs = [ ]
   
    a = html.find('img src=')
    while a != -1:
        
        b = html.find('.jpg', a, a+255)
        
        if b != -1:
            img_addrs.append(html[a+9:b+4])

        a = html.find('img src=', a + 9)


    return img_addrs


def save_img(img_addrs, name):
    j = 1
    for each in img_addrs:

        file_name = 'mm_' + str(name) + '_' +  str(j) + '.jpg'
        
        with open(file_name, 'wb') as f:
            html = url_open(each)
            f.write(html)
        j += 1

def download_mm(folder = 'ooxx', pages = 5):

    os.mkdir(folder)
    os.chdir(folder)

    url = 'http://jandan.net/ooxx/'
    page_num = int(get_page_num(url))

    for i in range(pages):

        #查找图片所在的网页

        page_num -= i
        page_url = url + 'page-' + str(page_num) + '#comments'

        #查找图片地址
        img_addrs = find_img(page_url)


        #保存图片
        save_img(img_addrs, i)



if __name__ == '__main__':

    download_mm()



报错信息:
>>>
=========== RESTART: C:\Users\Administrator\Desktop\download_mm.py ===========
Traceback (most recent call last):
  File "C:\Users\Administrator\Desktop\download_mm.py", line 81, in <module>
    download_mm()
  File "C:\Users\Administrator\Desktop\download_mm.py", line 75, in download_mm
    save_img(img_addrs, i)
  File "C:\Users\Administrator\Desktop\download_mm.py", line 51, in save_img
    html = url_open(each)
  File "C:\Users\Administrator\Desktop\download_mm.py", line 9, in url_open
    response = urllib.request.urlopen(req)
  File "C:\Python35\lib\urllib\request.py", line 163, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Python35\lib\urllib\request.py", line 466, in open
    response = self._open(req, data)
  File "C:\Python35\lib\urllib\request.py", line 489, in _open
    'unknown_open', req)
  File "C:\Python35\lib\urllib\request.py", line 444, in _call_chain
    result = func(*args)
  File "C:\Python35\lib\urllib\request.py", line 1324, in unknown_open
    raise URLError('unknown url type: %s' % type)
urllib.error.URLError: <urlopen error unknown url type: "http>
>>>

        

hldh214

在 open 之前 print 一下 url

tengfei8382

hldh214 发表于 2016-10-15 14:44
在 open 之前 print 一下 url

谢谢,尝试过在print找错,但是放的地方不对,各个open之前放一个立马找到错误原因....
find_img没有设置正确的中断条件,导致列表加入了别的元素...