|
|
马上注册，结交更多好友，享用更多功能^_^
您需要 登录 才可以下载或查看，没有账号？立即注册
x
本帖最后由 雨澄润 于 2026-9-26 16:44 编辑
这是一个爬虫程序，但爬虫爬取的内容与用户输入内容无关，希望能有大神能够帮忙解答一下。
有人能帮忙的话，真的万分感谢！！！
import tkinter as tk
from tkinter import messagebox, scrolledtext
import requests
from bs4 import BeautifulSoup
import sqlite3
import urllib.parse
import json
import time
# ==================== 数据库操作（完全保留原有逻辑） ====================
DB_NAME = "quotes.db"
def init_db():
"""初始化数据库，创建表；若旧表缺少 keyword 列则自动补上"""
conn = sqlite3.connect(DB_NAME)
cursor = conn.cursor()
cursor.execute(
"CREATE TABLE IF NOT EXISTS quotes ("
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
"text TEXT, "
"author TEXT, "
"keyword TEXT)"
)
cursor.execute("PRAGMA table_info(quotes)")
columns = [row[1] for row in cursor.fetchall()]
if "keyword" not in columns:
cursor.execute("ALTER TABLE quotes ADD COLUMN keyword TEXT")
conn.commit()
conn.close()
def save_to_db(data_list, keyword=""):
"""将爬取的数据批量存入数据库"""
if not data_list:
return 0
conn = sqlite3.connect(DB_NAME)
cursor = conn.cursor()
cursor.executemany(
"INSERT INTO quotes (text, author, keyword) VALUES (?, ?, ?)",
[(item["text"], item["author"], keyword) for item in data_list]
)
conn.commit()
conn.close()
return len(data_list)
def get_from_db():
"""从数据库读取所有数据"""
conn = sqlite3.connect(DB_NAME)
cursor = conn.cursor()
cursor.execute("SELECT text, author, keyword FROM quotes")
rows = cursor.fetchall()
conn.close()
return rows
# ==================== 统一请求头（模拟真实浏览器） ====================
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
}
# ==================== 源1：古诗文网（API接口，返回JSON） ====================
def crawl_from_gushiwen_api(keyword):
"""
古诗文网 - 通过API接口获取名句（JSON格式，无需HTML解析）
优势：结构稳定、中文无乱码、返回数据干净
"""
results = []
api_url = "https://api.gushiwen.cn/v3/gshiwang/shiju/search"
params = {
"key": keyword,
"page": 1,
"size": 20,
}
try:
resp = requests.get(api_url, headers=HEADERS, params=params, timeout=15)
# 使用 resp.text（自动编码检测），不手动 decode
resp.encoding = resp.apparent_encoding or 'utf-8'
data = resp.json()
if data and "data" in data and data["data"]:
for item in data["data"]:
text = item.get("title", "") or item.get("content", "")
author = item.get("author", "") or item.get("dynasty", "") + item.get("author", "")
if text and len(text.strip()) > 2:
results.append({
"text": text.strip(),
"author": author.strip() if author else "佚名"
})
except Exception as e:
raise Exception(f"古诗文网API请求失败: {str(e)}")
return results
# ==================== 源2：古诗文网（HTML页面备用） ====================
def crawl_from_gushiwen_html(keyword):
"""
古诗文网 - 通过HTML页面搜索备用
使用多个CSS选择器策略提高兼容性
"""
results = []
search_url = "https://so.gushiwen.cn/search/"
params = {
"value": keyword,
}
try:
resp = requests.get(search_url, headers=HEADERS, params=params, timeout=15)
# 关键修复：使用 resp.text 自动处理编码，避免中文乱码
resp.encoding = resp.apparent_encoding or 'utf-8'
html = resp.text
soup = BeautifulSoup(html, "html.parser")
# 策略1：尝试新的选择器 div.content > div.left > div.song > div.son > p
items = soup.select("div.content div.left div.song div.son p a")
if len(items) >= 2:
for i in range(0, len(items) - 1, 2):
text = items[i].get_text(strip=True)
author = items[i + 1].get_text(strip=True) if i + 1 < len(items) else ""
if text and len(text.strip()) > 2:
results.append({"text": text.strip(), "author": author.strip()})
# 策略2：尝试另一种选择器
if not results:
items = soup.select("div.sons p a")
if len(items) >= 2:
for i in range(0, len(items) - 1, 2):
text = items[i].get_text(strip=True)
author = items[i + 1].get_text(strip=True) if i + 1 < len(items) else ""
if text and len(text.strip()) > 2:
results.append({"text": text.strip(), "author": author.strip()})
# 策略3：尝试 div.left > div.son
if not results:
items = soup.select("div.left div.son p a")
if len(items) >= 2:
for i in range(0, len(items) - 1, 2):
text = items[i].get_text(strip=True)
author = items[i + 1].get_text(strip=True) if i + 1 < len(items) else ""
if text and len(text.strip()) > 2:
results.append({"text": text.strip(), "author": author.strip()})
except Exception as e:
raise Exception(f"古诗文网HTML请求失败: {str(e)}")
return results
# ==================== 源3：百度汉语（成语/词语释义） ====================
def crawl_from_baidu_hanyu(keyword):
"""
百度汉语 - 搜索成语/词语释义
作为古诗文网的备用源
"""
results = []
search_url = "https://dict.baidu.com/s"
params = {
"wd": keyword,
}
try:
resp = requests.get(search_url, headers=HEADERS, params=params, timeout=15)
resp.encoding = resp.apparent_encoding or 'utf-8'
html = resp.text
soup = BeautifulSoup(html, "html.parser")
# 尝试获取百度汉语的释义内容
# 策略：查找包含释义的div
definitions = soup.select("div.vc-paragraph-inner, div.vc-content, div.result")
for div in definitions:
text = div.get_text(strip=True)
if text and len(text) > 5 and keyword in text:
results.append({"text": text[:200], "author": "百度汉语"})
# 如果上面的选择器没匹配到，尝试更宽泛的搜索
if not results:
all_texts = soup.find_all(string=True)
for t in all_texts:
t_stripped = t.strip()
if keyword in t_stripped and len(t_stripped) > 10:
results.append({"text": t_stripped[:200], "author": "百度汉语"})
if len(results) >= 5:
break
except Exception as e:
raise Exception(f"百度汉语请求失败: {str(e)}")
return results
# ==================== 源4：一言API（随机名句） ====================
def crawl_from_hitokoto():
"""
一言API - 获取随机名句（不依赖关键词，作为兜底源）
"""
results = []
urls = [
"https://v1.hitokoto.cn/?c=a", # 诗词古典
"https://v1.hitokoto.cn/?c=f", # 哲学思想
"https://v1.hitokoto.cn/?c=d", # 原创
]
for url in urls:
try:
resp = requests.get(url, headers=HEADERS, timeout=10)
resp.encoding = resp.apparent_encoding or 'utf-8'
data = resp.json()
if "hitokoto" in data and data["hitokoto"]:
results.append({
"text": data["hitokoto"],
"author": data.get("from", "") or data.get("creator", "佚名")
})
except:
continue
return results
# ==================== 主爬虫函数（多源fallback + 关键词验证） ====================
def crawl_quotes(keyword=""):
"""
多源爬虫主函数：
1. 有关键词：优先古诗文网API -> HTML备用 -> 百度汉语
2. 无关键词：返回一言API随机名句
3. 自动编码检测，避免中文乱码
4. 关键词相关性验证，确保返回内容与搜索词相关
5. 去重、异常捕获
"""
keyword = keyword.strip()
results = []
seen = set()
def add_result(item):
"""添加结果并去重"""
text = item.get("text", "").strip()
author = item.get("author", "").strip()
if not text or len(text) < 3:
return
key = (text[:50], author[:20])
if key not in seen:
seen.add(key)
results.append({"text": text, "author": author})
def filter_by_keyword(items, kw):
"""过滤与关键词相关的结果"""
if not kw:
return items
filtered = []
kw_lower = kw.lower()
for item in items:
text = item.get("text", "")
author = item.get("author", "")
# 只要文本或作者中包含关键词就算相关
if kw_lower in text.lower() or kw_lower in author.lower():
filtered.append(item)
elif len(filtered) < 3:
# 允许部分匹配的结果（防止严格过滤导致无结果）
filtered.append(item)
return filtered if filtered else items # 兜底：至少返回一些结果
try:
if keyword:
# === 有关键词：多源搜索 ===
# 源1：古诗文网API
try:
api_results = crawl_from_gushiwen_api(keyword)
for item in api_results:
add_result(item)
except Exception:
pass
# 源2：古诗文网HTML备用
if len(results) < 3:
try:
html_results = crawl_from_gushiwen_html(keyword)
for item in html_results:
add_result(item)
except Exception:
pass
# 源3：百度汉语
if len(results) < 3:
try:
baidu_results = crawl_from_baidu_hanyu(keyword)
for item in baidu_results:
add_result(item)
except Exception:
pass
# 关键词相关性过滤
results = filter_by_keyword(results, keyword)
else:
# === 无关键词：返回随机名句 ===
results = crawl_from_hitokoto()
# 如果一言API也失败，尝试古诗文网默认名句
if not results:
try:
api_results = crawl_from_gushiwen_api("")
for item in api_results:
add_result(item)
results = api_results
except Exception:
pass
except Exception as e:
raise Exception(f"爬虫主流程异常: {str(e)}")
# 最终去重
final_results = []
final_seen = set()
for item in results:
key = item["text"][:50]
if key not in final_seen:
final_seen.add(key)
final_results.append(item)
return final_results
# ==================== GUI界面 ====================
init_db()
root = tk.Tk()
root.title("简易AI 8.0【多源爬虫 · 中文关键词搜索 · 修复版】")
root.geometry("750x620")
# ---------- 功能一：用户输入获取区 ----------
frame_input = tk.LabelFrame(root, text="用户输入获取（支持中文，如：理想、青春、奋斗、爱情）", padx=10, pady=10)
frame_input.pack(padx=10, pady=5, fill="x")
entry = tk.Entry(frame_input, width=45)
entry.pack(side="left", padx=5)
def get_input():
content = entry.get()
if content:
messagebox.showinfo("输入结果", f"你输入了：{content}")
else:
messagebox.showwarning("提示", "输入框为空！")
btn_get = tk.Button(frame_input, text="获取输入", command=get_input)
btn_get.pack(side="left", padx=5)
# ---------- 功能二：爬虫爬取与存储区 ----------
frame_crawl = tk.LabelFrame(root, text="爬虫爬取与存储（多源fallback机制）", padx=10, pady=10)
frame_crawl.pack(padx=10, pady=5, fill="both", expand=True)
btn_frame = tk.Frame(frame_crawl)
btn_frame.pack(pady=5)
btn_crawl = tk.Button(btn_frame, text="爬取并存储", width=14)
btn_crawl.pack(side="left", padx=5)
btn_show = tk.Button(btn_frame, text="查看数据库", width=14)
btn_show.pack(side="left", padx=5)
tk.Label(frame_crawl, text="爬取结果：", font=("微软雅黑", 10, "bold")).pack(anchor="w", padx=5)
# 使用 scrolledtext 替代 Text，支持自动滚动
text_area = scrolledtext.ScrolledText(frame_crawl, width=85, height=20, font=("微软雅黑", 10))
text_area.pack(padx=5, pady=5, fill="both", expand=True)
# ---------- 功能三：退出按钮 ----------
frame_exit = tk.Frame(root)
frame_exit.pack(pady=10)
def on_exit():
root.destroy()
btn_exit = tk.Button(frame_exit, text="退出", command=on_exit)
btn_exit.pack()
# ==================== 按钮回调函数 ====================
def on_crawl():
user_input = entry.get().strip()
btn_crawl.config(state="disabled", text="爬取中...")
root.update()
# 清空结果区
text_area.delete("1.0", tk.END)
# 显示搜索信息
if user_input:
text_area.insert(tk.END, f"===== 搜索关键词：{user_input} =====\n")
text_area.insert(tk.END, f"===== 多源搜索策略：古诗文网API → 古诗文网HTML → 百度汉语 =====\n\n")
else:
text_area.insert(tk.END, "===== 未输入关键词，抓取随机名句 =====\n")
text_area.insert(tk.END, f"===== 源：一言API + 古诗文网 =====\n\n")
# 执行爬取（带超时和异常处理）
try:
data = crawl_quotes(keyword=user_input)
except Exception as e:
text_area.insert(tk.END, f"【错误】爬虫执行失败：{str(e)}\n")
text_area.insert(tk.END, "请检查网络连接后重试。\n")
messagebox.showerror("爬取出错", f"爬虫执行失败：{str(e)}\n请检查网络连接！")
btn_crawl.config(state="normal", text="爬取并存储")
return
# 处理结果
if data:
count = save_to_db(data, keyword=user_input)
text_area.insert(tk.END, f"===== 共爬取并存储 {count} 条数据到数据库 =====\n\n")
for i, item in enumerate(data, 1):
text_area.insert(tk.END, f"【{i}】{item['text']}\n")
text_area.insert(tk.END, f" —— {item['author']}\n\n")
# 根据结果数量给出不同反馈
if len(data) >= 5:
messagebox.showinfo("成功", f"爬取完成！共获取 {count} 条数据（多源合并），已存储到数据库。")
elif len(data) >= 1:
messagebox.showinfo("部分成功", f"爬取完成！共获取 {count} 条数据，已存储到数据库。\n如结果不够精确，请尝试更换关键词。")
else:
messagebox.showwarning("结果有限", "未获取到足够数据，请检查网络或更换关键词！")
else:
# 无结果时的智能提示
if user_input:
text_area.insert(tk.END, f"未找到与「{user_input}」直接匹配的内容。\n")
text_area.insert(tk.END, "可能原因：\n")
text_area.insert(tk.END, "1. 网络连接不稳定，部分数据源未返回结果\n")
text_area.insert(tk.END, "2. 关键词较为生僻，尝试更通用的词语\n")
text_area.insert(tk.END, "3. 建议尝试：理想、青春、奋斗、思念、爱情、人生、月亮、花等常见主题\n")
messagebox.showwarning("无结果",
f"未找到与「{user_input}」匹配的内容。\n"
"建议：尝试更通用的关键词，如「理想」「青春」「奋斗」「思念」等。\n"
"请检查网络连接后重试。")
else:
text_area.insert(tk.END, "未能抓取到名句，请检查网络连接后重试。\n")
messagebox.showwarning("无结果", "未抓取到任何数据，请检查网络连接！")
btn_crawl.config(state="normal", text="爬取并存储")
def on_show():
rows = get_from_db()
if not rows:
messagebox.showwarning("提示", "数据库为空，请先爬取数据！")
return
text_area.delete("1.0", tk.END)
text_area.insert(tk.END, f"===== 数据库中共有 {len(rows)} 条数据 =====\n\n")
for i, (text, author, keyword) in enumerate(rows, 1):
tag_info = f"（搜索关键词：{keyword}）" if keyword else ""
text_area.insert(tk.END, f"【{i}】{text}\n")
text_area.insert(tk.END, f" —— {author} {tag_info}\n\n")
btn_crawl.config(command=on_crawl)
btn_show.config(command=on_show)
root.mainloop()
|
|
( 粤ICP备18085999号-1 | 粤公网安备 44051102000585号)
狗仔卡
发表于 2026-9-26 14:40:18
置顶卡
千斤顶
显身卡
楼主