lda分析 用jieba分词但是做不出来，报错了，求助
报错,找不到原因File "D:\python370\lib\site-packages\jieba\_compat.py", line 79, in strdecode
sentence = sentence.decode('utf-8')
AttributeError: 'int' object has no attribute 'decode'
"""
---------------------------------
test: rzbbzr
test1: PyCharm
test2: LDA20220212.py
test3: 2022-02-12 11:38
Pyinstaller -FLDA20220212.py
py转jupyter %load xxxx.py
pip install -i https://pypi.doubanio.com/simple/ 包名
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple +包名
"""
import os
import pandas as pd
import re
import jieba
import jieba.posseg as psg
#######预处理
output_path = 'D:/lda/data/result'
file_path = 'D:/lda/data'
os.chdir(file_path)
data = pd.read_excel("D:/lda/data/data.xlsx")
os.chdir(output_path)
dic_file = "D:/lda/data/dict.txt"
stop_file = "D:/lda/data/stopwords.txt"
def chinese_word_cut(mytext):
jieba.load_userdict(dic_file)
jieba.initialize()
try:
stopword_list = open(stop_file, encoding='utf-8')
except:
stopword_list = []
print("error in stop_file")
stop_list = []
flag_list = ['n', 'nz', 'vn']
for line in stopword_list:
line = re.sub(u'\n|\\r', '', line)
stop_list.append(line)
word_list = []
# jieba分词
seg_list = psg.cut(mytext)
for seg_word in seg_list:
# word = re.sub(u'[^\u4e00-\u9fa5]','',seg_word.word)
# seg_word = str(seg_word).strip()
print(type(seg_word))
word = seg_word.word
find = 0
print(type(word))
for stop_word in stop_list:
if stop_word == word or len(word) < 2:# this word is stopword
find = 1
break
if find == 0 and seg_word.flag in flag_list:
word_list.append(word)
return (" ").join(word_list)
data["content_cutted"] = data.content.apply(chinese_word_cut)
#######LDA分析
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
def print_top_words(model, feature_names, n_top_words):
tword = []
for topic_idx, topic in enumerate(model.components_):
print("Topic #%d:" % topic_idx)
topic_w = " ".join( for i in topic.argsort()[:-n_top_words - 1:-1]])
tword.append(topic_w)
print(topic_w)
return tword
n_features = 1000# 提取1000个特征词语
tf_vectorizer = CountVectorizer(strip_accents='unicode',
max_features=n_features,
stop_words='english',
max_df=0.5,
min_df=10)
tf = tf_vectorizer.fit_transform(data.content_cutted)
n_topics = 8
lda = LatentDirichletAllocation(n_components=n_topics, max_iter=50,
learning_method='batch',
learning_offset=50,
# doc_topic_prior=0.1,
# topic_word_prior=0.01,
random_state=0)
lda.fit(tf)
###########每个主题对应词语
n_top_words = 25
tf_feature_names = tf_vectorizer.get_feature_names()
topic_word = print_top_words(lda, tf_feature_names, n_top_words)
###########输出每篇文章对应主题
import numpy as np
topics = lda.transform(tf)
topic = []
for t in topics:
topic.append(list(t).index(np.max(t)))
data['topic'] = topic
data.to_excel("data_topic.xlsx", index=False)
topics# 0 1 2
###########可视化
import pyLDAvis
import pyLDAvis.sklearn
pyLDAvis.enable_notebook()
pic = pyLDAvis.sklearn.prepare(lda, tf, tf_vectorizer)
pyLDAvis.display(pic)
pyLDAvis.save_html(pic, 'lda_pass' + str(n_topics) + '.html')
# 去工作路径下找保存好的html文件
# pyLDAvis.show(pic)
###########困惑度
import matplotlib.pyplot as plt
plexs = []
n_max_topics = 16
for i in range(1, n_max_topics):
print(i)
lda = LatentDirichletAllocation(n_components=i, max_iter=50,
learning_method='batch',
learning_offset=50, random_state=0)
lda.fit(tf)
plexs.append(lda.perplexity(tf))
n_t = 15# 区间最右侧的值。注意：不能大于n_max_topics
x = list(range(1, n_t))
plt.plot(x, plexs)
plt.xlabel("number of topics")
plt.ylabel("perplexity")
plt.show()
用的下面的数据
链接：https://pan.baidu.com/s/1OkzDW_fXarDOGQg3VOYb3w
提取码：1111
问题出在jieba那里 , 谢谢
报错贴全一点，把完整的回溯过程贴上来 isdkz 发表于 2022-2-12 21:41
报错贴全一点，把完整的回溯过程贴上来
Traceback (most recent call last):
File "C:/Users/Administrator/PycharmProjects/pythonProject2/venv/lda/LDA20220212.py", line 67, in <module>
data["content_cutted"] = data.content.apply(chinese_word_cut)
File "D:\python370\lib\site-packages\pandas\core\series.py", line 4213, in apply
mapped = lib.map_infer(values, f, convert=convert_dtype)
File "pandas\_libs\lib.pyx", line 2403, in pandas._libs.lib.map_infer
File "C:/Users/Administrator/PycharmProjects/pythonProject2/venv/lda/LDA20220212.py", line 46, in chinese_word_cut
for seg_word in seg_list:
File "D:\python370\lib\site-packages\jieba\posseg\__init__.py", line 294, in cut
for w in dt.cut(sentence, HMM=HMM):
File "D:\python370\lib\site-packages\jieba\posseg\__init__.py", line 249, in cut
for w in self.__cut_internal(sentence, HMM=HMM):
File "D:\python370\lib\site-packages\jieba\posseg\__init__.py", line 217, in __cut_internal
sentence = strdecode(sentence)
File "D:\python370\lib\site-packages\jieba\_compat.py", line 79, in strdecode
sentence = sentence.decode('utf-8')
AttributeError: 'int' object has no attribute 'decode'
就上面这些,不知道应该改哪里 本帖最后由 isdkz 于 2022-2-12 22:41 编辑
在第67行前面加上这句试试
data.fillna('', inplace=True) data["content_cutted"] = data.content.apply(chinese_word_cut)
这里错了，整形没有decode方法。 isdkz 发表于 2022-2-12 22:10
在第67行前面加上这句试试
没起作用,还是报那个错误 swanseabrian 发表于 2022-2-12 22:50
没起作用,还是报那个错误
好吧 z5560636 发表于 2022-2-12 22:15
这里错了，整形没有decode方法。
那这里要怎么改一下,我不会改还,谢谢 方便把另外两个文件放上来吗？不然不好帮你调试
页:
[1]