[已解决]爬取豆瓣动作类电影报错

surf588

请教各位，爬取豆瓣动作类电影，运行后报错，代码如下：
import urllib.parse
import urllib.request

def create_request(page):
    base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&start=0&limit=20'

    data = {
        'start':(page - 1) * 20,
        'limit':20
    }

    data = urllib.parse.urlencode(data)

    url = base_url + data

    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36'
    }

    request = urllib.request.Request(url=url,headers=headers)
    return request

def get_content(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content

def down_load(page,content):
    with open('douban_' + str(page) +'.json','w',encoding='utf-8') as fp:
        fp.write(content)

if __name__ == '__main__':
    start_page = int(input('请输入起始页码'))
    end_page = int(input('请输入结束页码'))

    for page in range(start_page,end_page+1):
        request = create_request(page)
        content = get_content(request)
        down_load(page,content)
报错类型
Traceback (most recent call last):
  File "C:\Users\zing\PycharmProjects\pythonProject\get请求1th.py", line 51, in <module>
    content = get_content(request)
  File "C:\Users\zing\PycharmProjects\pythonProject\get请求1th.py", line 37, in get_content
    response = urllib.request.urlopen(request)
  File "D:\python\lib\urllib\request.py", line 214, in urlopen
    return opener.open(url, data, timeout)
  File "D:\python\lib\urllib\request.py", line 523, in open
    response = meth(req, response)
  File "D:\python\lib\urllib\request.py", line 632, in http_response
    response = self.parent.error(
  File "D:\python\lib\urllib\request.py", line 561, in error
    return self._call_chain(*args)
  File "D:\python\lib\urllib\request.py", line 494, in _call_chain
    result = func(*args)
  File "D:\python\lib\urllib\request.py", line 641, in http_error_default
    raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 500: INTERNAL SERVER ERROR

挠头中，向前辈们请教

最佳答案

月排行榜 / 总排行榜

isdkz

2022-5-11 13:48:07

start 和 limit 参数重复了，你的 data 那里有 start 和 limit 就不要在 base_url 那里写了

import urllib.parse
import urllib.request

def create_request(page):
    base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&'     # 注意这里，删掉 start 和 limit

    data = {
        'start':(page - 1) * 20,
        'limit':20
    }

    data = urllib.parse.urlencode(data)

    url = base_url + data
    print(url)
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36'
    }

    request = urllib.request.Request(url=url,headers=headers)
    return request

def get_content(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content

def down_load(page,content):
    with open('douban_' + str(page) +'.json','w',encoding='utf-8') as fp:
        fp.write(content)

if __name__ == '__main__':
    start_page = int(input('请输入起始页码'))
    end_page = int(input('请输入结束页码'))

    for page in range(start_page,end_page+1):
        request = create_request(page)
        content = get_content(request)
        down_load(page,content)

跳转到最佳答案楼层

isdkz

start 和 limit 参数重复了，你的 data 那里有 start 和 limit 就不要在 base_url 那里写了

import urllib.parse
import urllib.request

def create_request(page):
    base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&'     # 注意这里，删掉 start 和 limit

    data = {
        'start':(page - 1) * 20,
        'limit':20
    }

    data = urllib.parse.urlencode(data)

    url = base_url + data
    print(url)
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36'
    }

    request = urllib.request.Request(url=url,headers=headers)
    return request

def get_content(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content

def down_load(page,content):
    with open('douban_' + str(page) +'.json','w',encoding='utf-8') as fp:
        fp.write(content)

if __name__ == '__main__':
    start_page = int(input('请输入起始页码'))
    end_page = int(input('请输入结束页码'))

    for page in range(start_page,end_page+1):
        request = create_request(page)
        content = get_content(request)
        down_load(page,content)