求大神帮我看看这段爬豆瓣的代码 没报错但是爬出来的db文件是空的
from bs4 import BeautifulSoup
import re
import urllib.request,urllib.error
import xlwt
import sqlite3
def main():
baseuel="https://movie.douban.com/top250?start="
datalist = getdate(baseuel)
#savepath = "豆瓣电影top250.xls"
# saveDate(datalist,savepath)
dbpath = "movie.db"
saveDate2db(datalist,dbpath)
findlink = re.compile(r'<a href="(.*?)">')
findimgsrc = re.compile(r'<img.*src="(.*?)"',re.S)
findtitle = re.compile(r'<span class="title">(.*?)</span>')
findrating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')
findjudge = re.compile(r'<span>(\d*)人评价</span>')
findinq = re.compile(r'<span class="inq">(.*)</span>')
findbd = re.compile(r'<p class="">(.*?)</p>',re.S)
def getdate(baseurl):
datalist = []
for i in range(0,10):
url = baseurl + str(i*25)
html = askURL(url)
soup = BeautifulSoup(html,"html.parser")
for item in soup.find_all('div',class_="item"):
date = []
item = str(item)
link = re.findall(findlink,item).replace(":","")
date.append(link)
imgsrc = re.findall(findimgsrc,item).replace(":","")
date.append(imgsrc)
titles = re.findall(findtitle,item)
date.append(titles)
rating = re.findall(findrating,item)
date.append(rating)
judgenum = re.findall(findjudge,item)
date.append(judgenum)
inq = re.findall(findinq,item)
if len(inq) != 0:
inq = inq
date.append(inq)
else:
date.append(' ')
bd = re.findall(findbd,item)
date.append(bd.strip())
datalist.append(date)
return datalist
#得到一个指定url的网址内容
def askURL(url):
head = {
"一个地址"
}
request = urllib.request.Request(url,headers=head)
html = ""
try:
response = urllib.request.urlopen(request)
html = response.read().decode("utf-8")
except urllib.error.URLError as e:
if hasattr(e,"code"):
print(e.code)
return html
def saveDate(datalist,savepath):
book = xlwt.Workbook(encoding="utf-8", style_compression=0)
sheet = book.add_sheet('豆瓣电影TOP250', cell_overwrite_ok=True)
col = ("电影连接", "图片链接", "名字", "评分", "评价数", "概况", "相关信息")
for i in range(0,7):
sheet.write(0,i,col)
for i in range(0,250):
print("D%d条"%(i+1))
data = datalist
for j in range(0,7):
sheet.write(i + 1,j,data)
book.save(savepath)
def saveDate2db(datalist,dbpath):
init_db(dbpath)
conn = sqlite3.connect(dbpath)
cur = conn.cursor()
for data in datalist:
for index in range(len(data)):
if index == 3 or index == 4:
continue
data = '"'+str(data)+'“'
sql = '''
insert into movie250(
info_link,pic_link,name,score,rated,introduction,info)
values(%s)'''%",".join(data)
print(sql)
cur.execute(sql)
conn.commit()
cur.close()
conn.close()
def init_db(dbpath):
sql = '''
create table movie250
(
id integer primary key autoincrement,
info_link text,
pic_link text,
cname varchar,
score numeric,
rated numeric,
introduction text,
info text
)
''' #创建数据表
conn = sqlite3.connect(dbpath)
cursor = conn.cursor()
cursor.execute(sql)
conn.commit()
conn.close()
if __name__ == "__main__":
main() 你先打印一下爬取出来的数据如果有数据就是sql语句错了。
只会 mysql
页:
[1]