[已解决]爬虫遇到的小问题

代码小白liu

源代码如下：

1. import requests
   
2. from lxml import etree
   
3. import os
   
4. 
   
5. url = "https://www.ibswtan.com/0/425/"
   
6. headers = {
   
7.     "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"
   
8. }
   
9. 
   
10. response = requests.get(url,headers=headers)
    
11. response.encoding = response.apparent_encoding
    
12. html_text = response.text
    
13. html_tree = etree.HTML(html_text)
    
14. div_list = html_tree.xpath('//div[@id="list"]/dl/dd')
    
15. url_list = []
    
16. if not os.path.exists("./斗破苍穹"):
    
17.     os.mkdir("./斗破苍穹")
    
18. for dd in div_list:
    
19.     new_url = dd.xpath("./a/@href")[0]
    
20.     url_list.append(new_url)
    
21. 
    
22. for i in url_list:
    
23.     download_url = "https://www.ibswtan.com/0/425/" + i
    
24.     # print(download_url)
    
25.     r = requests.get(download_url, headers=headers)
    
26.     r.encoding = r.apparent_encoding
    
27.     detal_html = r.text
    
28.     detal_tree = etree.HTML(detal_html)
    
29.     book_name = detal_tree.xpath('//div[@class="bookname"]/h1/text()')[0]
    
30.     text = detal_tree.xpath('//div[@id="content"]/text()')[0]
    
31.     file_path = "./斗破苍穹/" + book_name + ".txt"
    
32.     with open(file_path, "w", encoding="utf-8") as f:
    
33.         f.write(text)
    
34.         print(book_name,'下载完毕')
    
35. 
    

复制代码

爬虫之后报错：

1. 四章完毕 下载完毕
   
2. 萧炎，我们的主角的图片哦~ 下载完毕
   
3. Traceback (most recent call last):
   
4.   File "E:/推土机/py-lzy/Requests-bs4-xpath/爬虫-笔趣阁-斗破苍穹.py", line 29, in <module>
   
5.     book_name = detal_tree.xpath('//div[@class="bookname"]/h1/text()')[0]
   
6. IndexError: list index out of range

复制代码

到报错的位置，我看了【土豆强力推荐——《》网游】这个页面的标题代码是一样的啊，单独爬取能爬取到数据的，为啥会超出索引那

最佳答案

月排行榜 / 总排行榜

大马强

2021-8-1 22:45:53

                               
登录/注册后可看大图

1. import requests
   
2. from lxml import etree
   
3. import os
   
4. import time
   
5. 
   
6. 
   
7. url = "https://www.ibswtan.com/0/425/"
   
8. headers = {
   
9.     "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"
   
10. }
    
11. 
    
12. response = requests.get(url, headers=headers)
    
13. response.encoding = response.apparent_encoding
    
14. html_text = response.text
    
15. html_tree = etree.HTML(html_text)
    
16. div_list = html_tree.xpath('//div[@id="list"]/dl/dd')
    
17. url_list = []
    
18. if not os.path.exists("./斗破苍穹"):
    
19.     # os.mkdir("./斗破苍穹")
    
20.     for dd in div_list:
    
21.         new_url = dd.xpath("./a/@href")[0]
    
22.         url_list.append(new_url)
    
23. 
    
24.     for i in url_list:
    
25.         download_url = "https://www.ibswtan.com/0/425/" + i
    
26.         print(download_url)
    
27.         time.sleep(1)  # 睡上1秒就可以了
    
28.         r = requests.get(download_url, headers=headers)
    
29.         r.encoding = r.apparent_encoding
    
30.         detal_html = r.text
    
31.         detal_tree = etree.HTML(detal_html)
    
32. 
    
33.         book_name = detal_tree.xpath(
    
34.             '//div[@class="bookname"]/h1/text()')[0]
    
35.         text = detal_tree.xpath('//div[@id="content"]/text()')[0]
    
36.         file_path = "./斗破苍穹/" + book_name + ".txt"
    
37.         with open(file_path, "w", encoding="utf-8") as f:
    
38.             f.write(text)
    
39.             print(book_name, '下载完毕')
    

复制代码

可能是请求的太频繁了，睡上1秒就应该没问题了

跳转到最佳答案楼层

wp231957

[0]都超出范围，明显是一个空列表

大马强

改了一点你的代码，运行是没有问题，但有一些章节就是爬不了

大马强

                               
登录/注册后可看大图

都是会断了几章

大马强

我发现了，你好像是被反爬了

大马强

                               
登录/注册后可看大图

1. import requests
   
2. from lxml import etree
   
3. import os
   
4. import time
   
5. 
   
6. 
   
7. url = "https://www.ibswtan.com/0/425/"
   
8. headers = {
   
9.     "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"
   
10. }
    
11. 
    
12. response = requests.get(url, headers=headers)
    
13. response.encoding = response.apparent_encoding
    
14. html_text = response.text
    
15. html_tree = etree.HTML(html_text)
    
16. div_list = html_tree.xpath('//div[@id="list"]/dl/dd')
    
17. url_list = []
    
18. if not os.path.exists("./斗破苍穹"):
    
19.     # os.mkdir("./斗破苍穹")
    
20.     for dd in div_list:
    
21.         new_url = dd.xpath("./a/@href")[0]
    
22.         url_list.append(new_url)
    
23. 
    
24.     for i in url_list:
    
25.         download_url = "https://www.ibswtan.com/0/425/" + i
    
26.         print(download_url)
    
27.         time.sleep(1)  # 睡上1秒就可以了
    
28.         r = requests.get(download_url, headers=headers)
    
29.         r.encoding = r.apparent_encoding
    
30.         detal_html = r.text
    
31.         detal_tree = etree.HTML(detal_html)
    
32. 
    
33.         book_name = detal_tree.xpath(
    
34.             '//div[@class="bookname"]/h1/text()')[0]
    
35.         text = detal_tree.xpath('//div[@id="content"]/text()')[0]
    
36.         file_path = "./斗破苍穹/" + book_name + ".txt"
    
37.         with open(file_path, "w", encoding="utf-8") as f:
    
38.             f.write(text)
    
39.             print(book_name, '下载完毕')
    

复制代码

可能是请求的太频繁了，睡上1秒就应该没问题了

suchocolate

我这运行正常输出。