不懂就问-爬虫相关

嗷呜呜呜

爬取豆瓣读书top250,为什么代码只爬到了第205就不爬了

1. import re
   
2. import requests
   
3. import csv
   
4. 
   
5. 
   
6. headers = {
   
7.     "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
   
8.     "Referer":"https://book.douban.com/top250?start=50"
   
9. }
   
10. 
    
11. books = []
    
12. for i in range(10):#共10页
    
13.     url = "https://book.douban.com/top250?start="+str(i*25)
    
14. 
    
15.     response = requests.get(url,headers=headers)
    
16.     text = response.content.decode("utf-8")
    
17. 
    
18. #所有书的名字
    
19.     names = re.findall(r'<div\sclass="pl2">.*?<a.*?>(.*?)</a>',text,re.DOTALL)
    
20.     # print(names)
    
21.     all_names = []
    
22.     for name in names:
    
23.         name = name.replace("\n", "").strip()
    
24.         beizhu  = re.findall(r"<span.*?>(.*?)</span>",name,re.DOTALL)
    
25.         name = re.sub(r"<span.*?>.*?</span>","",name,re.DOTALL).strip()
    
26.         if beizhu == []:
    
27.             name = name
    
28.         else:
    
29.             name = name+beizhu[0]
    
30.         all_names.append(name)
    
31. #所有书的其他名字
    
32.     other_names = re.findall(r'<div\sclass="pl2">.*?<a.*?>.*?</a>.*?<span.*?>(.*?)</span>',text,re.DOTALL)
    
33. #所有书的作者
    
34.     writers = re.findall(r'<div\sclass="pl2">.*?<p\sclass="pl">(.*?)</p>',text,re.DOTALL)
    
35. #所有书的评分
    
36.     scores = re.findall(r'<div\sclass="star clearfix">.*?<span\sclass="rating_nums">(.*?)</span>',text,re.DOTALL)
    
37. #所有书的评价人数
    
38.     all_people = []
    
39.     people = re.findall(r'<div\sclass="star clearfix">.*?<span\sclass="pl">(.*?)</span>',text,re.DOTALL)
    
40.     for each_people in people:
    
41.         each_people = each_people.replace("\n","").replace("(","").replace(")","").strip()
    
42.         all_people.append(each_people)
    
43. #所有书的优美句子
    
44.     nice_statments = re.findall(r'<p\sclass="quote"\sstyle=".*?">.*?<span.*?>(.*?)</span>',text,re.DOTALL)
    
45. 
    
46. #完成每本书对应数据的整合
    
47.     each_books = list(zip(all_names,other_names,writers,scores,all_people,nice_statments))
    
48.     for each_book in each_books:
    
49.         all_names,other_names,writers,scores,all_people,nice_statments=each_book
    
50.         book = {
    
51.             "name":all_names,
    
52.             "other_name":other_names,
    
53.             "writer":writers,
    
54.             "score":scores,
    
55.             "people":all_people,
    
56.             "nice_statment":nice_statments
    
57.         }
    
58.         books.append(book)
    
59.     # books.sort(key=lambda x:float(x["score"]),reverse = True)
    
60. 
    
61. header = ["name","other_name","writer","score","people","nice_statment"]
    
62. with open("doubandushu1.csv","w",encoding = "utf-8",newline="")as f:
    
63.     writer = csv.DictWriter(f,header)#请求csv标题
    
64.     writer.writeheader()#写入csv标题
    
65.     writer.writerows(books)#写入内容

复制代码

heqianqian

增加一行校验：
print(len(nice_statments))
得到结果
25
25
24
25
25
25
25
22
9
0
你nice_statments只有205个

嗷呜呜呜

heqianqian 发表于 2019-4-7 18:55
增加一行校验：
print(len(nice_statments))
得到结果

找到原因了,网页源代码的问题,不过还是很感谢