[已解决]仿照爬豆瓣top250，爬先知社区标题，作者时间

Thewhitecrow

import requests
import bs4
import re

def open_url(url):

    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}

    res = requests.get(url, headers=headers)
    return res

def find_title(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')

    #文章标题
    title = []
    targets = soup.find_all('p',class_='topic-summary')
    for each in targets:
        title.append(each.a.text)

    #资料（作者，类型，时间）
    messages = []
    targets = soup.find_all('p',class_='topic-info')
    for each in targets:
        try:
            messages.append(each.a.text.split('\n')[1].strip() + each.a.text.split('\n')[2].strip())
        except:
            continue

    result = []
    length = len(title)
    for i in range(length):
        result.append(title[i] + messages[i] + '\n')
    return result

    #找出一共多少页
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    depth = soup.find('li', class_='disable').previous_sibling.previous_sibling.text

    return int(depth)

def main():
    res = "https://xz.aliyun.com/"
    res = open_url(res)
    depth = find_depth(res)

    result = []
    for i in range(depth):
        url = res + '/?page=' + str(2 * i)
        res = open_url(url)
        result.extend(find_title(res))

        with open("先知.txt",'w',encoding='utf-8') as file:
            for each in result:
                file.write(each)
if __name__ == "__main__":
    main()

最佳答案

月排行榜 / 总排行榜

1q23w31

2020-8-5 14:15:47

Thewhitecrow 发表于 2020-8-4 18:21
报错AttributeError: 'NoneType' object has no attribute 'previous_sibling'，我感觉是一共多少页那里有 ...

1. import requests
   
2. import bs4
   
3. import re
   
4. 
   
5. def open_url(url):
   
6. 
   
7.     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
   
8. 
   
9.     res = requests.get(url, headers=headers)
   
10.     return res
    
11. 
    
12. def find_title(res):
    
13.     soup = bs4.BeautifulSoup(res.text,'html.parser')
    
14. 
    
15.     #文章标题
    
16.     title = []
    
17.     targets = soup.find_all('p',class_='topic-summary')
    
18.     for each in targets:
    
19.         title.append(each.a.text.strip('\n').strip(' '))
    
20. 
    
21.     #资料（作者，类型，时间）
    
22.     messages = []
    
23.     targets = soup.find_all('p',class_='topic-info')
    
24.     for each in targets:
    
25.             messages.append([each.a.text ,re.search(r"(\d{4}-\d{1,2}-\d{1,2})",each.text).group()])
    
26.     result = []
    
27.     length = len(title)
    
28.     for i in range(length):
    
29. 
    
30.         result.append([title[i],messages[i]])
    
31.     return result
    
32. 
    
33.     #找出一共多少页
    
34. def find_depth(res):
    
35.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
36.     depth = soup.select('#Wrapper > div > div.span10 > div > div > div.pagination.clearfix > ul > li:nth-child(2) > a')
    
37.     depth = re.search('\d{3}',str(depth))
    
38.     return int(depth.group())
    
39. 
    
40. def main():
    
41.     res = "https://xz.aliyun.com/"
    
42.     res1 = open_url(res)
    
43.     depth = find_depth(res1)
    
44. 
    
45.     result = []
    
46.     for i in range(1,depth):
    
47.         url = res + '/?page=' + str(2 * i)
    
48.         print('正在爬取第',i,'页')
    
49.         res2 = open_url(url)
    
50.         
    
51.         result.extend(find_title(res2))
    
52. 
    
53.         with open("先知.txt",'w',encoding='utf-8') as file:
    
54.             for each in result:
    
55.                 file.write(str(each))
    
56.                 file.write('\n')
    
57.                 file.write('=============================================')
    
58.                 file.write('\n')
    
59. if __name__ == "__main__":
    
60.     main()
    

复制代码

代码已改好

跳转到最佳答案楼层

1q23w31

Thewhitecrow 发表于 2020-8-5 14:53
老哥为啥我去复制代码，发现爬虫结果没爬干净，是从7.23开始爬，之前的没有爬到

1. import requests
   
2. import bs4
   
3. import re
   
4. 
   
5. def open_url(url):
   
6. 
   
7.     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
   
8. 
   
9.     res = requests.get(url, headers=headers)
   
10.     return res
    
11. 
    
12. def find_title(res):
    
13.     soup = bs4.BeautifulSoup(res.text,'html.parser')
    
14. 
    
15.     #文章标题
    
16.     title = []
    
17.     targets = soup.find_all('p',class_='topic-summary')
    
18.     for each in targets:
    
19.         title.append(each.a.text.strip('\n').strip(' '))
    
20. 
    
21.     #资料（作者，类型，时间）
    
22.     messages = []
    
23.     targets = soup.find_all('p',class_='topic-info')
    
24.     for each in targets:
    
25.             messages.append([each.a.text ,re.search(r"(\d{4}-\d{1,2}-\d{1,2})",each.text).group()])
    
26.     result = []
    
27.     length = len(title)
    
28.     for i in range(length):
    
29. 
    
30.         result.append([title[i],messages[i]])
    
31.     return result
    
32. 
    
33.     #找出一共多少页
    
34. def find_depth(res):
    
35.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
36.     depth = soup.select('#Wrapper > div > div.span10 > div > div > div.pagination.clearfix > ul > li:nth-child(2) > a')
    
37.     depth = re.search('\d{3}',str(depth))
    
38.     return int(depth.group())
    
39. 
    
40. def main():
    
41.     res = "https://xz.aliyun.com/"
    
42.     res1 = open_url(res)
    
43.     result = []
    
44.     result.extend(find_title(res1))
    
45.     depth = find_depth(res1)
    
46. 
    
47. 
    
48.     for i in range(1,depth):
    
49.         url = res + '/?page=' + str(2 * i)
    
50.         print('正在爬取第',i,'页')
    
51.         res2 = open_url(url)
    
52.         
    
53.         result.extend(find_title(res2))
    
54. 
    
55.         with open("先知.txt",'w',encoding='utf-8') as file:
    
56.             for each in result:
    
57.                 file.write(str(each))
    
58.                 file.write('\n')
    
59.                 file.write('=============================================')
    
60.                 file.write('\n')
    
61. if __name__ == "__main__":
    
62.       main()

复制代码

ok之前没把第一页加进去

Thewhitecrow

报错AttributeError: 'NoneType' object has no attribute 'previous_sibling'，我感觉是一共多少页那里有点问题，希望前辈们指点下

suchocolate

从网页html看，class为disable的li没有前兄弟节点，所以报错说没有。



bs不熟，用xpath写了一个简单的：

1. 
   
2. import requests
   
3. from lxml import etree
   
4. import re
   
5. 
   
6. 
   
7. def main():
   
8.     url = 'https://xz.aliyun.com/'
   
9.     headers = {'user-agent': 'firefox'}
   
10.     r = requests.get(url, headers=headers)
    
11.     html = etree.HTML(r.text)
    
12.     trs = html.xpath('//tr')
    
13.     for item in trs:
    
14.         title = item.xpath('normalize-space(./td/p[@class="topic-summary"]/a[@class="topic-title"]/text())')
    
15.         info = item.xpath('./td/p[@class="topic-info"]//text()')
    
16.         print('标题：', title)
    
17.         print('作者:', info[1])
    
18.         print('类型：', info[3])
    
19.         print('发布时间：', re.findall('\d{4}(?:-\d{2}){2}', info[4])[0])
    
20.         print('='*50)
    
21. 
    
22. if __name__ == '__main__':
    
23.         main()
    

复制代码

陈尚涵

suchocolate 发表于 2020-8-5 10:41
从网页html看，class为disable的li没有前兄弟节点，所以报错说没有。

你这个pycharm有点亮眼啊

1q23w31

Thewhitecrow 发表于 2020-8-4 18:21
报错AttributeError: 'NoneType' object has no attribute 'previous_sibling'，我感觉是一共多少页那里有 ...

1. import requests
   
2. import bs4
   
3. import re
   
4. 
   
5. def open_url(url):
   
6. 
   
7.     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
   
8. 
   
9.     res = requests.get(url, headers=headers)
   
10.     return res
    
11. 
    
12. def find_title(res):
    
13.     soup = bs4.BeautifulSoup(res.text,'html.parser')
    
14. 
    
15.     #文章标题
    
16.     title = []
    
17.     targets = soup.find_all('p',class_='topic-summary')
    
18.     for each in targets:
    
19.         title.append(each.a.text.strip('\n').strip(' '))
    
20. 
    
21.     #资料（作者，类型，时间）
    
22.     messages = []
    
23.     targets = soup.find_all('p',class_='topic-info')
    
24.     for each in targets:
    
25.             messages.append([each.a.text ,re.search(r"(\d{4}-\d{1,2}-\d{1,2})",each.text).group()])
    
26.     result = []
    
27.     length = len(title)
    
28.     for i in range(length):
    
29. 
    
30.         result.append([title[i],messages[i]])
    
31.     return result
    
32. 
    
33.     #找出一共多少页
    
34. def find_depth(res):
    
35.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
36.     depth = soup.select('#Wrapper > div > div.span10 > div > div > div.pagination.clearfix > ul > li:nth-child(2) > a')
    
37.     depth = re.search('\d{3}',str(depth))
    
38.     return int(depth.group())
    
39. 
    
40. def main():
    
41.     res = "https://xz.aliyun.com/"
    
42.     res1 = open_url(res)
    
43.     depth = find_depth(res1)
    
44. 
    
45.     result = []
    
46.     for i in range(1,depth):
    
47.         url = res + '/?page=' + str(2 * i)
    
48.         print('正在爬取第',i,'页')
    
49.         res2 = open_url(url)
    
50.         
    
51.         result.extend(find_title(res2))
    
52. 
    
53.         with open("先知.txt",'w',encoding='utf-8') as file:
    
54.             for each in result:
    
55.                 file.write(str(each))
    
56.                 file.write('\n')
    
57.                 file.write('=============================================')
    
58.                 file.write('\n')
    
59. if __name__ == "__main__":
    
60.     main()
    

复制代码

代码已改好

Thewhitecrow

suchocolate 发表于 2020-8-5 10:41
从网页html看，class为disable的li没有前兄弟节点，所以报错说没有。

老哥我刚才发现你的的class原因，也改了。还是有点问题，debug的时候在有多少页这个地方，语法有问题

1. import requests
   
2. import bs4
   
3. import re
   
4. 
   
5. def open_url(url):
   
6. 
   
7.     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
   
8. 
   
9.     res = requests.get(url, headers=headers)
   
10.     return res
    
11. 
    
12. def find_title(res):
    
13.     soup = bs4.BeautifulSoup(res.text,'html.parser')
    
14. 
    
15.     #文章标题
    
16.     title = []
    
17.     targets = soup.find_all('p',class_='topic-summary')
    
18.     for each in targets:
    
19.         title.append(each.a.text)
    
20. 
    
21.     #资料（作者，类型，时间）
    
22.     messages = []
    
23.     targets = soup.find_all('p',class_='topic-info')
    
24.     for each in targets:
    
25.         try:
    
26.             messages.append(each.p.text.split('\n')[1].strip() + each.p.text.split('\n')[2].strip())
    
27.         except:
    
28.             continue
    
29. 
    
30.     result = []
    
31.     length = len(title)
    
32.     for i in range(length):
    
33.         result.append(title[i] + messages[i] + '\n')
    
34.     return result
    
35. 
    
36.     #找出一共多少页
    
37. def find_depth(res):
    
38.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
39.     depth = soup.find('li', class_='disabled').previous_sibling.previous_sibling.text
    
40. 
    
41.     return int(depth)
    
42. 
    
43. def main():
    
44.     host = "https://xz.aliyun.com/"
    
45.     res = open_url(host)
    
46.     depth = find_depth(res)
    
47. 
    
48.     result = []
    
49.     for i in range(depth):
    
50.         url = host + '/?page=' + str(2 + i)
    
51.         res = open_url(url)
    
52.         result.extend(find_title(res))
    
53. 
    
54.         with open("先知.txt",'w',encoding='utf-8') as file:
    
55.             for each in result:
    
56.                 file.write(each)
    
57. 
    
58. if __name__ == "__main__":
    
59.     main()

复制代码

Thewhitecrow

1q23w31 发表于 2020-8-5 14:15
代码已改好

老哥你这真的强，我好像明白了，谢谢老哥

Thewhitecrow

suchocolate 发表于 2020-8-5 10:41
从网页html看，class为disable的li没有前兄弟节点，所以报错说没有。

老哥你这我去学习下，非常感谢哥哥

Thewhitecrow

1q23w31 发表于 2020-8-5 14:15
代码已改好

老哥为啥我去复制代码，发现爬虫结果没爬干净，是从7.23开始爬，之前的没有爬到

1q23w31

Thewhitecrow 发表于 2020-8-5 14:53
老哥为啥我去复制代码，发现爬虫结果没爬干净，是从7.23开始爬，之前的没有爬到

稍等，我改一下代码

Thewhitecrow

1q23w31 发表于 2020-8-5 15:01
ok之前没把第一页加进去

老哥感谢，够自己去反思学习了