最新的源代码
请问谁有最新的爬去新闻的源代码呢 import requestsfrom bs4 import BeautifulSoup
import re
import json
from datetime import datetime
import pandas
#取得评论数的函数
def getCommentCounts(newsurl):
m = re.search('doc-i(.+).shtml',newsurl)
newsid = m.group(1)
commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=sh&newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'
comments = requests.get(commentURL.format(newsid))
comments.encoding ='utf-8'
jd = json.loads(comments.text.strip('var data='))
return (jd['result']['count']['total'])
#在正文页面取得包含标题、新闻来源、时间、正文、编辑姓名、评论数的一个字典result
def getNewsDetail(newsurl):
result = {}
res = requests.get(newsurl)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text,'html.parser')
result['title'] = soup.select('#artibodyTitle').text
result['newssource'] = soup.select('.time-source a').text
timesource = soup.select('.time-source').contents.strip()
result['dt'] = datetime.strptime(timesource,'%Y年%m月%d日%H:%M')
result['article'] = ' '.join(])
result['editor'] = soup.select('.article-editor').text.lstrip('责任编辑：')
result['comments'] = getCommentCounts(newsurl)
return result
'''
#在新闻标题列表页面如何取得第一次加载的时间、标题以及链接
res_1 = requests.get('http://news.sina.com.cn/society/')
res_1.encoding = 'utf-8'
soup =BeautifulSoup(res_1.text,'html.parser')
for news in soup.select('.news-item'):
if len(news.select('h2'))>0:
h2 = news.select('h2').text
time = news.select('.time').text
a = news.select('a')['href']
print (time,h2,a)
'''
#在新闻标题列表页面如何取得滚动后加载的所有新闻链接，并调用getNewsDetail(newsurl)函数
def parseListLinks(url):
res = requests.get(url)
jd = json.loads(res.text.lstrip(' newsloadercallback(').rstrip(');'))
newsdetails = []
for each in jd['result']['data']:
newsdetails.append(getNewsDetail(each['url']))#这里调用了上面的getNewsDetail(newsurl)函数
return newsdetails
url = 'http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=shxw&cat_2==zqsk||=qwys||=shwx||=fz-shyf&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page={}'
news_total = []
for i in range(1,6):
newsurl = url.format(i)
newsary = parseListLinks(newsurl)
news_total.extend(newsary)
df = pandas.DataFrame(news_total)
df.to_excel('news.xlsx')
import sqlite3
with sqlite3.connect('news.sqlite')as db:
df.to_sql('news',con = db)
import sqlite3
with sqlite3.connect('news.sqlite')as db:
df2 = pandas.read_sql_query('SELECT * FROM NEWS',con = db) 本帖最后由 六小鸭 于 2020-3-30 17:43 编辑
我啥都没事说 六小鸭 发表于 2020-3-30 17:38
话说你真要走闲鱼卖号
废话,我感觉我着迷在闲鱼论坛上,浪费时间了 六小鸭 发表于 2020-3-30 17:38
话说你真要走闲鱼卖号
六小鸭 发表于 2020-3-30 17:37
Traceback (most recent call last):
File "D:\desktop\study\py\1.py", line 62, in <module>
newsary = parseListLinks(newsurl)
File "D:\desktop\study\py\1.py", line 50, in parseListLinks
jd = json.loads(res.text.lstrip(' newsloadercallback(').rstrip(');'))
File "D:\python\lib\json\__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "D:\python\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "D:\python\lib\json\decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
错误 一个账号 发表于 2020-3-30 17:42
这只是好奇 一个账号 发表于 2020-3-30 17:42
好吧 https://blog.csdn.net/nice_xp/article/details/73038599
自己学习
页:
[1]