[已解决]豆瓣爬TOP250的问题，我用的是xpath

suchocolate

我发重了，内容在楼下。

suchocolate

quote有的没有内容，缺少东西容易造成数据不齐。这个适合把每部的li各自过xpath：

1. 
   
2. import requests
   
3. from lxml import etree
   
4. import json
   
5. 
   
6. 
   
7. def main():
   
8.     headers = {
   
9.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36',
   
10.         'Cookie': 'll="108314"; bid=_LAWkGgi1Js; ct=y; __yadk_uid=89YZlsdVstEBGt8rNf4innJ79Hi3mosn; __gads=ID=92c5f47901f551a0-22f530c6f2c40037:T=1606531304:RT=1606531304:S=ALNI_Mae6Hq1eo6RkR_9HRim4ml_HoEYRg; __utmc=30149280; __utmz=30149280.1606727878.7.3.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmc=223695111; __utmz=223695111.1606727878.7.3.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _vwo_uuid_v2=D30C7E4BAE9F96458B12515507FA532DC|b7a3a014f000afb51be010698c310b59; __utma=30149280.1401849356.1606469592.1606797311.1606801152.10; __utmb=30149280.0.10.1606801152; __utma=223695111.352245982.1606469592.1606797311.1606801152.10; __utmb=223695111.0.10.1606801152; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1606801152%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3D9C40AoCTrSNIfGCVB3nwTgy1k-fMWS3e2H0qSiDrPdvEzWRxS1RD8d937yYKIajE%26wd%3D%26eqid%3D8b4f698a0001959a000000035fc4b8c1%22%5D; _pk_ses.100001.4cf6=*; ap_v=0,6.0; _pk_id.100001.4cf6=3e8dbc93b7d06862.1606469591.10.1606801589.1606798873.'
    
11.     }
    
12.     movies = []
    
13.     num = 1
    
14.     for i in range(10):
    
15.         n = i * 25
    
16.         url = f'https://movie.douban.com/top250?start={n}'
    
17.         r = requests.get(url=url, headers=headers)
    
18.         html = etree.HTML(r.text)
    
19.         lis = html.xpath('//ol/li')
    
20.         for li in lis:
    
21.             rank = li.xpath('./div/div[1]/em/text()')[0]
    
22.             name = li.xpath('./div/div[2]/div[1]/a/span[1]/text()')[0]
    
23.             # print(name)
    
24.             director = li.xpath('normalize-space(./div/div[2]/div[2]/p/text()[1])')
    
25.             # print(director)
    
26.             score = li.xpath('./div/div[2]/div[2]/div/span[2]/text()')[0]
    
27.             # print(score)
    
28.             quote = li.xpath('./div/div[2]/div[2]/p[2]/span/text()')
    
29.             if not quote:
    
30.                 quote = ['暂无']
    
31.             m = {'rank': rank, 'name': name, 'director': director, 'score': score, 'quote': quote[0]}
    
32.             movies.append(m)
    
33.             print(f'已添加{name}, 共添加{num}部。')
    
34.             num += 1
    
35.     with open('movie.json', 'w', encoding='utf-8') as f:
    
36.         f.write(json.dumps(movies, indent=2, ensure_ascii=False))
    
37. 
    
38. 
    
39. if __name__ == '__main__':
    
40.     main()

复制代码