为什么爬取百度百科的时候会出现错误

RIXO · 发表于 2018-9-24 00:02:25

马上注册，结交更多好友，享用更多功能^_^

您需要登录才可以下载或查看，没有账号？立即注册

x

本帖最后由 RIXO 于 2018-9-24 09:40 编辑

呃，看书籍编的一个爬百度百科的代码，本来没有爬取成功，但是我在git上面看到一个爬取成功的，怀着好奇的心理就下载了，结果他的代码能用。。。。。
看了一下，发现在爬百度百科的时候，他使用的是urllib.request.urlopen(url)连header都没有加，但是能爬
结果我的用requests库，就会出现requests.exceptions.TooManyRedirects: Exceeded 30 redirects. 这个错误
求个解答！
呃，贴代码

import requests
import urlparse2,re,html_codec
from bs4 import BeautifulSoup
class UrlManager():
def __init__(self):
self.new_urls = set()
self.old_urls = set()
def has_new_url(self): #是否有待获取的url
return self.new_url_size != 0
def get_new_url(self): #获取一个新的url
new_url = self.new_urls.pop()
self.old_urls.add(new_url)
return new_url
def add_new_url(self,url): #加入新的url
if url is None:
return
if url not in self.new_urls and url not in self.old_urls:
self.new_urls.add(url)
def add_new_urls(self,urls):
if urls is None or len(urls) == 0:
return
for url in urls:
self.add_new_urls(url)
def new_url_size(self):
return len(self.new_urls)
def old_url_size(self):
return len(self.old_urls)
class HtmlDownloader(): #返回html内容
def download(self,url):
if url is None:
return None
headers = {
'Host' : 'baike.baidu.com',
'User_Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36',
'Upgrade-Insecure-Requests': '1'
}
r = requests.get(url,headers = headers,allow_redirects=False)
if r.status_code == 200:
r,encoding = 'utf-8'
return r.text
return None
class HtmlParser():
def parser(self,page_url,html_cont):
if page_url is None or html_cont is None :
return
soup = BeautifulSoup(html_cont,'html.parser',from_encoding='utf-8')
new_urls = self._get_new_urls(page_url,soup)
new_data = self._get_new_data(page_url,soup)
def _get_new_urls(self,page_url,soup): #解析出url
new_urls = set()
links = soup.find_all('a',href = re.compile(r'/view/\d+\.htm'))
for link in links:
new_url = link('href')
new_full_url = urlparse2.urljoin(page_url,new_url)
new_urls.add(new_full_url)
return new_urls
def _get_new_data(self,page_url,soup): #解析出数据
data = {}
data['url'] = page_url
title = soup.find('dd',class_ = 'lemmaWgt-lemmaTitle-title').find('h1')
data['title'] = title.get_text()
summary = soup.find('div',class_ = 'lemma-summary' )
data['summary'] = summary.get_text()
return data
class DataOutput():
def __init__(self):
self.datas = []
def store_data(self,data):
if data is None:
return
self.datas.append(data)
def output_html(self):
fout = html_codec.open('baike.html','w',encoding = 'utf-8')
fout.write('<html>')
fout.write('<body>')
fout.write('<table>')
for data in self.datas:
fout.wite('<tr>')
fout.write('<td>%s</td>'%data['url'])
fout.write('<td>%s</td>'%data['title'])
fout.write('<td>%s</td>'%data['summary'])
fout.write('</tr>')
self.datas.remove(data)
fout.write('</table>')
fout.write('</body>')
fout.write('</html>')
fout.close()
class SpiderMan():
def __init__(self):
self.manager = UrlManager()
self.downloader = HtmlDownloader()
self.parser = HtmlParser()
self.output = DataOutput()
def craw(self,root_url):
self.manager.add_new_url(root_url)
while (self.manager.has_new_url() and self.manager.old_url_size()<1):
#try:
new_url = self.manager.get_new_url()
html = self.downloader.download(new_url)
new_urls,data = self.parser.parser(new_url,html)
self.manager.add_new_urls(new_urls)
self.output.store_data(data)
print('已经抓取了%s个链接'%self.manager.old_url_size())
#except Exception as e:
# print('crawl failed',e)
self.output.output_html()
if __name__ == '__main__':
spider_man = SpiderMan()
spider_man.craw('https://baike.baidu.com/item/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB')

复制代码

塔利班 · 发表于 2018-9-24 07:45:07

代码呢

孤独的嫖客 · 发表于 2018-9-24 08:52:52

测试采集网址:https://baike.baidu.com/vbaike/
测试结果:
QQ截图20180924085159.jpg

RIXO · 发表于 2018-9-24 09:40:26

塔利班发表于 2018-9-24 07:45
代码呢

呃，贴上了

塔利班 · 发表于 2018-9-24 10:04:50

fout.wite('<tr>')拼写
parse2和html_codec库没装，也没用过= =

RIXO · 发表于 2018-9-24 10:08:37

塔利班发表于 2018-9-24 10:04
fout.wite('')拼写
parse2和html_codec库没装，也没用过= =

我也没用过，抄的代码，hhh，问题还没到那里呢，在htmldownload的模块那里就出问题了
那里用requests库获取网址的时候会出现链接超出的情况

塔利班 · 发表于 2018-9-24 10:08:57

安装了库运行也是各种错误= =

塔利班 · 发表于 2018-9-24 10:10:40

一般说的超链接超出都是cookies没维持住，你用session试试

RIXO · 发表于 2018-9-24 10:13:03

本帖最后由 RIXO 于 2018-9-24 10:16 编辑

塔利班发表于 2018-9-24 10:08
安装了库运行也是各种错误= =

再给你看看git上面找到的。

# coding:utf8
class UrlManager(object):
def __init__(self):
self.new_urls = set()
self.old_urls = set()
def add_new_url(self,url): #向管理器中添加一个新的url
if url is None:
return
if url not in self.new_urls and url not in self.old_urls:
self.new_urls.add(url)
def add_new_urls(self,urls): #向管理器中添加批量的url
if urls is None or len(urls) == 0:
return
for url in urls:
self.add_new_url(url)
def has_new_url(self): #判断管理器中是否有新的待爬取的url
return len(self.new_urls) != 0
def get_new_url(self): #从url管理器中获取一个新的待爬取的url
new_url = self.new_urls.pop()
self.old_urls.add(new_url)
return new_url
# coding:utf8
import requests
class HtmlDownloader(object):
def download(self, url):
if url is None:
return None
response = requests.get(url)
#response = urllib.request.urlopen(url)
#equest = urllib.request.Request(url)
#response = urllib.request.urlopen(request,context = context)
if response.getcode() != 200:
return None
return response.read()
# coding:utf8
from bs4 import BeautifulSoup
import re
import urllib.request
class HtmlParser(object):
def _get_new_urls(self, page_url, soup):
new_urls = set()
# /item/%E8%AE... 或 /item/GPL
links = soup.find_all('a', href=re.compile(r"/item/.*"))
for link in links:
new_url = link['href']
new_full_url = urllib.parse.urljoin(page_url,new_url)
new_urls.add(new_full_url)
return new_urls
def _get_new_data(self, page_url, soup):
res_data = {}
#url
res_data['url'] = page_url
# <dd class="lemmaWgt-lemmaTitle-title"> <h1>Python</h1>
title_node = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find("h1")
res_data['title'] = title_node.get_text()
# <div class="lemma-summary" label-module="lemmaSummary">
summary_node = soup.find('div', class_="lemma-summary")
res_data['summary'] = summary_node.get_text()
return res_data
def parse(self,page_url,html_cont):
if page_url is None or html_cont is None:
return
soup = BeautifulSoup(html_cont,'html.parser',from_encoding='utf-8')
new_urls = self._get_new_urls(page_url,soup)
new_data = self._get_new_data(page_url,soup)
return new_urls,new_data
# coding:utf8
import os
class HtmlOutputer(object):
def __init__(self):
self.datas = []
def collect_data(self,data):
if data is None:
return
self.datas.append(data)
def output_html(self):
fout = open('output.html', 'w', encoding='utf-8')
fout.write("<html>")
fout.write("<body>")
fout.write("<table>")
# python默认编码：ascii，这里要改成utf-8
for data in self.datas:
fout.write("<tr>")
fout.write("<td>%s</td>" %data['url'])
fout.write("<td>%s</td>" %data['title'])
fout.write("<td>%s</td>" %data['summary'])
fout.write("</tr>")
fout.write("</table>")
fout.write("</body>")
fout.write("</html>")
fout.close()
# coding:utf8
import url_manager, html_downloader, html_parser,html_outputer,os
class SpiderMain(object):
def __init__(self):
self.urls = url_manager.UrlManager()
self.downloader = html_downloader.HtmlDownloader()
self.parser = html_parser.HtmlParser()
self.outputer = html_outputer.HtmlOutputer()
def craw(self, root_url):
count = 1
self.urls.add_new_url(root_url)
while self.urls.has_new_url():
#try:
new_url = self.urls.get_new_url()
print ('craw %d : %s' % (count, new_url))
html_cont = self.downloader.download(new_url)
new_urls,new_data = self.parser.parse(new_url,html_cont)
self.urls.add_new_urls(new_urls)
self.outputer.collect_data(new_data)
print(os.getcwd())
if count == 1000:
break
count = count + 1
#except Exception as r:
#print ('craw failed',r)
self.outputer.output_html()
if __name__=="__main__":
# http://baike.baidu.com/view/21087.htm 或 https://baike.baidu.com/item/Python/407313
root_url = "http://baike.baidu.com/view/21087.htm"
obj_spider = SpiderMain()
obj_spider.craw(root_url)
print('finish')

复制代码

这个和我从书上抄的差不多，但区别在于，他下载网页的时候用urlopen干的，就能下（我在那里改成了requests然后就不能了），由于是从多个文件里面复制过来的，有些多余的语句可以删掉

RIXO · 发表于 2018-9-24 10:18:44

塔利班发表于 2018-9-24 10:10
一般说的超链接超出都是cookies没维持住，你用session试试

session试过了，出现了一样的问题，但是他用urlopen能用，就是比较慢，我都先百度了一遍问题解决，然后再来搞的，最近发现问题在百度上面都找不到相同的，有点绝望

塔利班 · 发表于 2018-9-24 10:18:47

不看了，爬虫我是菜鸡

账号		自动登录	找回密码
密码			立即注册