[已解决]仿照爬豆瓣top250，爬先知社区标题，作者时间

Thewhitecrow · 发表于 2020-8-4 18:20:14

马上注册，结交更多好友，享用更多功能^_^

您需要登录才可以下载或查看，没有账号？立即注册

x

import requests
import bs4
import re

def open_url(url):

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}

res = requests.get(url, headers=headers)
return res

def find_title(res):
soup = bs4.BeautifulSoup(res.text,'html.parser')

#文章标题
title = []
targets = soup.find_all('p',class_='topic-summary')
for each in targets:
      title.append(each.a.text)

#资料（作者，类型，时间）
messages = []
targets = soup.find_all('p',class_='topic-info')
for each in targets:
      try:
         messages.append(each.a.text.split('\n')[1].strip() + each.a.text.split('\n')[2].strip())
      except:
         continue

result = []
length = len(title)
for i in range(length):
      result.append(title[i] + messages[i] + '\n')
return result

#找出一共多少页
def find_depth(res):
soup = bs4.BeautifulSoup(res.text, 'html.parser')
depth = soup.find('li', class_='disable').previous_sibling.previous_sibling.text

return int(depth)

def main():
res = "https://xz.aliyun.com/"
res = open_url(res)
depth = find_depth(res)

result = []
for i in range(depth):
      url = res + '/?page=' + str(2 * i)
      res = open_url(url)
      result.extend(find_title(res))

      with open("先知.txt",'w',encoding='utf-8') as file:
         for each in result:
            file.write(each)
if __name__ == "__main__":
main()

最佳答案

月排行榜 / 总排行榜

1q23w31

2020-8-5 14:15:47

Thewhitecrow 发表于 2020-8-4 18:21
报错AttributeError: 'NoneType' object has no attribute 'previous_sibling'，我感觉是一共多少页那里有 ...

import requests
import bs4
import re

def open_url(url):

    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}

    res = requests.get(url, headers=headers)
    return res

def find_title(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')

    #文章标题
    title = []
    targets = soup.find_all('p',class_='topic-summary')
    for each in targets:
        title.append(each.a.text.strip('\n').strip(' '))

    #资料（作者，类型，时间）
    messages = []
    targets = soup.find_all('p',class_='topic-info')
    for each in targets:
            messages.append([each.a.text ,re.search(r"(\d{4}-\d{1,2}-\d{1,2})",each.text).group()])
    result = []
    length = len(title)
    for i in range(length):

        result.append([title[i],messages[i]])
    return result

    #找出一共多少页
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    depth = soup.select('#Wrapper > div > div.span10 > div > div > div.pagination.clearfix > ul > li:nth-child(2) > a')
    depth = re.search('\d{3}',str(depth))
    return int(depth.group())

def main():
    res = "https://xz.aliyun.com/"
    res1 = open_url(res)
    depth = find_depth(res1)

    result = []
    for i in range(1,depth):
        url = res + '/?page=' + str(2 * i)
        print('正在爬取第',i,'页')
        res2 = open_url(url)
        
        result.extend(find_title(res2))

        with open("先知.txt",'w',encoding='utf-8') as file:
            for each in result:
                file.write(str(each))
                file.write('\n')
                file.write('=============================================')
                file.write('\n')
if __name__ == "__main__":
    main()

代码已改好

跳转到最佳答案楼层

1q23w31 · 发表于 2020-8-5 15:01:56

Thewhitecrow 发表于 2020-8-5 14:53
老哥为啥我去复制代码，发现爬虫结果没爬干净，是从7.23开始爬，之前的没有爬到

import requests
import bs4
import re

def open_url(url):

    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}

    res = requests.get(url, headers=headers)
    return res

def find_title(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')

    #文章标题
    title = []
    targets = soup.find_all('p',class_='topic-summary')
    for each in targets:
        title.append(each.a.text.strip('\n').strip(' '))

    #资料（作者，类型，时间）
    messages = []
    targets = soup.find_all('p',class_='topic-info')
    for each in targets:
            messages.append([each.a.text ,re.search(r"(\d{4}-\d{1,2}-\d{1,2})",each.text).group()])
    result = []
    length = len(title)
    for i in range(length):

        result.append([title[i],messages[i]])
    return result

    #找出一共多少页
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    depth = soup.select('#Wrapper > div > div.span10 > div > div > div.pagination.clearfix > ul > li:nth-child(2) > a')
    depth = re.search('\d{3}',str(depth))
    return int(depth.group())

def main():
    res = "https://xz.aliyun.com/"
    res1 = open_url(res)
    result = []
    result.extend(find_title(res1))
    depth = find_depth(res1)


    for i in range(1,depth):
        url = res + '/?page=' + str(2 * i)
        print('正在爬取第',i,'页')
        res2 = open_url(url)
        
        result.extend(find_title(res2))

        with open("先知.txt",'w',encoding='utf-8') as file:
            for each in result:
                file.write(str(each))
                file.write('\n')
                file.write('=============================================')
                file.write('\n')
if __name__ == "__main__":
      main()

ok之前没把第一页加进去

Thewhitecrow · 发表于 2020-8-4 18:21:19

报错AttributeError: 'NoneType' object has no attribute 'previous_sibling'，我感觉是一共多少页那里有点问题，希望前辈们指点下

suchocolate · 发表于 2020-8-5 10:41:40

本帖最后由 suchocolate 于 2020-8-5 10:44 编辑

从网页html看，class为disable的li没有前兄弟节点，所以报错说没有。

Annotation 2020-08-05 100836.png

bs不熟，用xpath写了一个简单的：

import requests
from lxml import etree
import re


def main():
    url = 'https://xz.aliyun.com/'
    headers = {'user-agent': 'firefox'}
    r = requests.get(url, headers=headers)
    html = etree.HTML(r.text)
    trs = html.xpath('//tr')
    for item in trs:
        title = item.xpath('normalize-space(./td/p[@class="topic-summary"]/a[@class="topic-title"]/text())')
        info = item.xpath('./td/p[@class="topic-info"]//text()')
        print('标题：', title)
        print('作者:', info[1])
        print('类型：', info[3])
        print('发布时间：', re.findall('\d{4}(?:-\d{2}){2}', info[4])[0])
        print('='*50)

if __name__ == '__main__':
        main()

陈尚涵 · 发表于 2020-8-5 13:57:34

suchocolate 发表于 2020-8-5 10:41
从网页html看，class为disable的li没有前兄弟节点，所以报错说没有。

你这个pycharm有点亮眼啊

1q23w31 · 发表于 2020-8-5 14:15:47

这个最佳答案由 1q23w31 给出，感谢 1q23w31 的回答。

单击隐藏图章

Thewhitecrow 发表于 2020-8-4 18:21
报错AttributeError: 'NoneType' object has no attribute 'previous_sibling'，我感觉是一共多少页那里有 ...

import requests
import bs4
import re

def open_url(url):

    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}

    res = requests.get(url, headers=headers)
    return res

def find_title(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')

    #文章标题
    title = []
    targets = soup.find_all('p',class_='topic-summary')
    for each in targets:
        title.append(each.a.text.strip('\n').strip(' '))

    #资料（作者，类型，时间）
    messages = []
    targets = soup.find_all('p',class_='topic-info')
    for each in targets:
            messages.append([each.a.text ,re.search(r"(\d{4}-\d{1,2}-\d{1,2})",each.text).group()])
    result = []
    length = len(title)
    for i in range(length):

        result.append([title[i],messages[i]])
    return result

    #找出一共多少页
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    depth = soup.select('#Wrapper > div > div.span10 > div > div > div.pagination.clearfix > ul > li:nth-child(2) > a')
    depth = re.search('\d{3}',str(depth))
    return int(depth.group())

def main():
    res = "https://xz.aliyun.com/"
    res1 = open_url(res)
    depth = find_depth(res1)

    result = []
    for i in range(1,depth):
        url = res + '/?page=' + str(2 * i)
        print('正在爬取第',i,'页')
        res2 = open_url(url)
        
        result.extend(find_title(res2))

        with open("先知.txt",'w',encoding='utf-8') as file:
            for each in result:
                file.write(str(each))
                file.write('\n')
                file.write('=============================================')
                file.write('\n')
if __name__ == "__main__":
    main()

代码已改好

Thewhitecrow · 发表于 2020-8-5 14:42:36

suchocolate 发表于 2020-8-5 10:41
从网页html看，class为disable的li没有前兄弟节点，所以报错说没有。

老哥我刚才发现你的的class原因，也改了。还是有点问题，debug的时候在有多少页这个地方，语法有问题

import requests
import bs4
import re

def open_url(url):

    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}

    res = requests.get(url, headers=headers)
    return res

def find_title(res):
    soup = bs4.BeautifulSoup(res.text,'html.parser')

    #文章标题
    title = []
    targets = soup.find_all('p',class_='topic-summary')
    for each in targets:
        title.append(each.a.text)

    #资料（作者，类型，时间）
    messages = []
    targets = soup.find_all('p',class_='topic-info')
    for each in targets:
        try:
            messages.append(each.p.text.split('\n')[1].strip() + each.p.text.split('\n')[2].strip())
        except:
            continue

    result = []
    length = len(title)
    for i in range(length):
        result.append(title[i] + messages[i] + '\n')
    return result

    #找出一共多少页
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    depth = soup.find('li', class_='disabled').previous_sibling.previous_sibling.text

    return int(depth)

def main():
    host = "https://xz.aliyun.com/"
    res = open_url(host)
    depth = find_depth(res)

    result = []
    for i in range(depth):
        url = host + '/?page=' + str(2 + i)
        res = open_url(url)
        result.extend(find_title(res))

        with open("先知.txt",'w',encoding='utf-8') as file:
            for each in result:
                file.write(each)

if __name__ == "__main__":
    main()

Thewhitecrow · 发表于 2020-8-5 14:44:11

1q23w31 发表于 2020-8-5 14:15
代码已改好

老哥你这真的强，我好像明白了，谢谢老哥

Thewhitecrow · 发表于 2020-8-5 14:44:44

suchocolate 发表于 2020-8-5 10:41
从网页html看，class为disable的li没有前兄弟节点，所以报错说没有。

老哥你这我去学习下，非常感谢哥哥

Thewhitecrow · 发表于 2020-8-5 14:53:13

1q23w31 发表于 2020-8-5 14:15
代码已改好

老哥为啥我去复制代码，发现爬虫结果没爬干净，是从7.23开始爬，之前的没有爬到

1q23w31 · 发表于 2020-8-5 14:58:51

Thewhitecrow 发表于 2020-8-5 14:53
老哥为啥我去复制代码，发现爬虫结果没爬干净，是从7.23开始爬，之前的没有爬到

稍等，我改一下代码

Thewhitecrow · 发表于 2020-8-5 15:05:30

1q23w31 发表于 2020-8-5 15:01
ok之前没把第一页加进去

老哥感谢，够自己去反思学习了

账号		自动登录	找回密码
密码			立即注册