python爬虫豆瓣250部电影出现indexError
本帖最后由 猛男必看 于 2021-1-20 16:50 编辑各位大神们好，在学习小甲鱼的爬虫实例。出现了两个错误，一是在运行时，红色代码出现了IndexError: list index out of range。第二个错误是紫色代码出现了TypeError: 'NoneType' object is not iterable。新手上路，还请多多指教
import requests
import bs4
import re
def open_url(url):
headers={'Host':'movie.douban.com',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36'}
res=requests.get(url,headers=headers)
return res
def find_movies(res):
soup=bs4.BeautifulSoup(res.text,'html.parser')
movies=[]
targets=soup.find_all('div',class_='hd')
for each in targets:
movies.append(each.a.span.text)
ranks=[]
targets=soup.find_all('span',class_='rating_num')
for each in targets:
ranks.append('评分：%s'%each.text)
messages=[]
targets=soup.find_all('div',class_='bd')
for each in targets:
try:
messages.append(each.p.text.split('\n').srtip()+each.p.text.split('\n').strip())
except:
continue
length=len(movies)
result=[]
for i in range(length):
result.append(movies+ranks+messages+'\n')
return result
def find_depth(res):
soup=bs4.BeautifulSoup(res.text,'html.parser')
depth=soup.find('span',class_='next').previous_sibling.previous_sibling.text
return int(depth)
def main():
host='https://movie.douban.com/top250'
res=open_url(host)
depth=find_depth(res)
result=[]
for i in range(depth):
url=host+'/?start='+str(25*i)
res=open_url(url)
result.extend(find_movies(res))
with open('豆瓣TOP250电影.txt','w',encoding='utf-8') as f:
for each in result:
f.write(each)
if __name__=='__main__':
main() 红色部分的代码是那样的吗？也许你想说的是result.append(movies+ranks+messages+'\n')，message列表是空列表 代码贴出来的时候用代码块包一下啊，这代码怎么看啊。 EthanHsiung 发表于 2021-1-20 17:38
红色部分的代码是那样的吗？也许你想说的是result.append(movies+ranks+messages+'\n')，message列表是空列 ...
没错，是这样的，我往里复制代码的时候，中括号和里面的内容死活复制不上去了，不知道为啥 import requests
from lxml import etree
def main():
url = 'https://movie.douban.com/top250'
headers = {'user-agent': 'firefox'}
params = {'start': ''}
movies = []
for x in range(10):
params['start'] = x
r = requests.get(url, headers=headers, params=params)
html = etree.HTML(r.text)
lis = html.xpath('//ol[@class="grid_view"]/li')
for li in lis:
m_name = li.xpath('.//div[@class="hd"]/a/span/text()')
m_rank = li.xpath('.//div[@class="pic"]/em/text()')
m_score = li.xpath('.//div[@class="star"]/span/text()')
m_info = li.xpath('normalize-space(.//div[@class="bd"]/p/text())')
movies.append()
print(movies)
if __name__ == '__main__':
main()
页:
[1]