求助个爬豆瓣top的问题 照着书上来的

laogesix

from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import requests
import os


def open_url(url):
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"}
    res = requests.get(url,headers = headers)
    return res


def find_movies(res):
    soup = BeautifulSoup(res.text,"html.parser")
    #movie_name
    movie = []
    targets = soup.find_all("div",class_="hd")
    for each in targets:
        movie.append(each.a.span.text)
    #movie_grade
    grade = []
    targets = soup.find_all("span", class_="rating_num")
    for each in targets:
        grade.append("评分：%s" % each.text)
    #movie_info
    info = []
    targets = soup.find_all("div", class_="bd")
    for each in targets:
        try:
            info.append(each.p.text.spilt("\n")[1].strip()+each.p.text.spilt("\n")[2].strip())
        except:
            continue
    result = []
    length = len(movie)
    for i in range(length):
        result.append(movie[i] + grade[i] + info[i]+'\n')
    return result


#find all pages
def find_depth(res):
    soup = BeautifulSoup(res.text, "html.parser")
    depth = soup.find("span",class_='next').previous_sibling.previous_sibling.text
    return int(depth)
def main():
    host = "https://movie.douban.com/top250"
    res = open_url(host)
    depth = find_depth(res)

    result = []
    for i in range(depth):
        url = host +"/?start=" + str(25 * i)
        res = open_url(url)
        result.extend(find_movies(res))

    with open("Douban top250 movie info.txt","w",encoding="utf-8") as f:
        for each in result:
            f.write(each)

if __name__ == "__main__":
    main()


上面是代码，下面是错误信息。

Traceback (most recent call last):
  File "C:/**/douban2.py", line 68, in <module>
    main()
  File "C:/**/douban2.py", line 61, in main
    result.extend(find_movies(res))
  File "C:/**/douban2.py", line 43, in find_movies
    result.append(movie[i] + grade[i] + info[i]+'\n')
IndexError: list index out of range

不知道为什么会错误，，越界也没有啊


求高人指点迷津

六小鸭

1. import os
   
2. import re
   
3. import time
   
4. import requests
   
5. from bs4 import BeautifulSoup
   
6. 
   
7. 
   
8. def download(url, page):
   
9.     print(f"正在爬取：{url}")
   
10.     html = requests.get(url).text   # 这里不加text返回<Response [200]>
    
11.     soup = BeautifulSoup(html, 'html.parser')
    
12.     lis = soup.select("ol li")
    
13.     for li in lis:
    
14.         index = li.find('em').text
    
15.         title = li.find('span', class_='title').text
    
16.         rating = li.find('span', class_='rating_num').text
    
17.         strInfo = re.search("(?<=<br/>).*?(?=<)", str(li.select_one(".bd p")), re.S | re.M).group().strip()
    
18.         infos = strInfo.split('/')
    
19.         year = infos[0].strip()
    
20.         area = infos[1].strip()
    
21.         type = infos[2].strip()
    
22.         write_fo_file(index, title, rating, year, area, type)
    
23.     page += 25
    
24.     if page < 250:
    
25.         time.sleep(2)
    
26.         download(f"https://movie.douban.com/top250?start={page}&filter=", page)
    
27. 
    
28. 
    
29. def write_fo_file(index, title, rating, year, area, type):
    
30.     f = open('movie_top250.csv', 'a')
    
31.     f.write(f'{index},{title},{rating},{year},{area},{type}\n')
    
32.     f.closed
    
33. 
    
34. 
    
35. def main():
    
36.     if os.path.exists('movie_top250.csv'):
    
37.         os.remove('movie_top250.csv')
    
38. 
    
39.     url = 'https://movie.douban.com/top250'
    
40.     download(url, 0)
    
41.     print("爬取完毕。")
    
42. 
    
43. 
    
44. if __name__ == '__main__':
    
45.     main()

复制代码

laogesix

六小鸭 发表于 2020-4-14 16:44

大佬我想知道我这哪错了

六小鸭

没报错啊

laogesix

六小鸭 发表于 2020-4-14 16:50
没报错啊

                               
登录/注册后可看大图

我俩个编辑器都报错

六小鸭

在我这里没报错

六小鸭

哦不我错了

六小鸭

我研究研究

laogesix

六小鸭 发表于 2020-4-14 16:55
哦不我错了

我说的我的代码有问题

六小鸭

laogesix 发表于 2020-4-14 16:59
我说的我的代码有问题

对不起，能成自己的了

六小鸭

你可以百度一下
我帮你@点大佬
@不二如是 @一个账号

laogesix

六小鸭 发表于 2020-4-14 17:07
你可以百度一下
我帮你@点大佬
@不二如是 @一个账号

这个应该百度不到，谢谢了