lda分析 用jieba分词但是做不出来，报错了，求助

swanseabrian

报错,找不到原因
File "D:\python370\lib\site-packages\jieba\_compat.py", line 79, in strdecode
    sentence = sentence.decode('utf-8')
AttributeError: 'int' object has no attribute 'decode'

1. """
   
2. ---------------------------------
   
3. test: rzbbzr
   
4. test1: PyCharm
   
5. test2: LDA20220212.py
   
6. test3: 2022-02-12 11:38
   
7. Pyinstaller -F  LDA20220212.py
   
8. py转jupyter %load xxxx.py
   
9. pip install -i https://pypi.doubanio.com/simple/ 包名
   
10. pip install -i https://pypi.tuna.tsinghua.edu.cn/simple +包名
    
11. """
    
12. import os
    
13. import pandas as pd
    
14. import re
    
15. import jieba
    
16. import jieba.posseg as psg
    
17. 
    
18. #######预处理
    
19. 
    
20. output_path = 'D:/lda/data/result'
    
21. file_path = 'D:/lda/data'
    
22. os.chdir(file_path)
    
23. data = pd.read_excel("D:/lda/data/data.xlsx")
    
24. os.chdir(output_path)
    
25. dic_file = "D:/lda/data/dict.txt"
    
26. stop_file = "D:/lda/data/stopwords.txt"
    
27. 
    
28. 
    
29. def chinese_word_cut(mytext):
    
30.     jieba.load_userdict(dic_file)
    
31.     jieba.initialize()
    
32.     try:
    
33.         stopword_list = open(stop_file, encoding='utf-8')
    
34.     except:
    
35.         stopword_list = []
    
36.         print("error in stop_file")
    
37.     stop_list = []
    
38.     flag_list = ['n', 'nz', 'vn']
    
39.     for line in stopword_list:
    
40.         line = re.sub(u'\n|\\r', '', line)
    
41.         stop_list.append(line)
    
42. 
    
43.     word_list = []
    
44.     # jieba分词
    
45.     seg_list = psg.cut(mytext)
    
46.     for seg_word in seg_list:
    
47.         # word = re.sub(u'[^\u4e00-\u9fa5]','',seg_word.word)
    
48.         # seg_word = str(seg_word).strip()
    
49.         print(type(seg_word))
    
50.         word = seg_word.word
    
51. 
    
52. 
    
53.         find = 0
    
54.         print(type(word))
    
55.         for stop_word in stop_list:
    
56. 
    
57. 
    
58.             if stop_word == word or len(word) < 2:  # this word is stopword
    
59.                 find = 1
    
60. 
    
61.                 break
    
62.         if find == 0 and seg_word.flag in flag_list:
    
63.             word_list.append(word)
    
64.     return (" ").join(word_list)
    
65. 
    
66. 
    
67. data["content_cutted"] = data.content.apply(chinese_word_cut)
    
68. 
    
69. #######LDA分析
    
70. 
    
71. from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
    
72. from sklearn.decomposition import LatentDirichletAllocation
    
73. 
    
74. 
    
75. def print_top_words(model, feature_names, n_top_words):
    
76.     tword = []
    
77.     for topic_idx, topic in enumerate(model.components_):
    
78.         print("Topic #%d:" % topic_idx)
    
79.         topic_w = " ".join([feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]])
    
80.         tword.append(topic_w)
    
81.         print(topic_w)
    
82.     return tword
    
83. 
    
84. 
    
85. n_features = 1000  # 提取1000个特征词语
    
86. tf_vectorizer = CountVectorizer(strip_accents='unicode',
    
87.                                 max_features=n_features,
    
88.                                 stop_words='english',
    
89.                                 max_df=0.5,
    
90.                                 min_df=10)
    
91. tf = tf_vectorizer.fit_transform(data.content_cutted)
    
92. 
    
93. n_topics = 8
    
94. lda = LatentDirichletAllocation(n_components=n_topics, max_iter=50,
    
95.                                 learning_method='batch',
    
96.                                 learning_offset=50,
    
97.                                 #                                 doc_topic_prior=0.1,
    
98.                                 #                                 topic_word_prior=0.01,
    
99.                                 random_state=0)
    
100. lda.fit(tf)
     
101. 
     
102. ###########每个主题对应词语
     
103. n_top_words = 25
     
104. tf_feature_names = tf_vectorizer.get_feature_names()
     
105. topic_word = print_top_words(lda, tf_feature_names, n_top_words)
     
106. 
     
107. ###########输出每篇文章对应主题
     
108. import numpy as np
     
109. 
     
110. topics = lda.transform(tf)
     
111. topic = []
     
112. for t in topics:
     
113.     topic.append(list(t).index(np.max(t)))
     
114. data['topic'] = topic
     
115. data.to_excel("data_topic.xlsx", index=False)
     
116. topics[0]  # 0 1 2
     
117. 
     
118. ###########可视化
     
119. 
     
120. import pyLDAvis
     
121. import pyLDAvis.sklearn
     
122. 
     
123. pyLDAvis.enable_notebook()
     
124. pic = pyLDAvis.sklearn.prepare(lda, tf, tf_vectorizer)
     
125. pyLDAvis.display(pic)
     
126. pyLDAvis.save_html(pic, 'lda_pass' + str(n_topics) + '.html')
     
127. # 去工作路径下找保存好的html文件
     
128. # pyLDAvis.show(pic)
     
129. 
     
130. 
     
131. ###########困惑度
     
132. import matplotlib.pyplot as plt
     
133. 
     
134. plexs = []
     
135. n_max_topics = 16
     
136. for i in range(1, n_max_topics):
     
137.     print(i)
     
138.     lda = LatentDirichletAllocation(n_components=i, max_iter=50,
     
139.                                     learning_method='batch',
     
140.                                     learning_offset=50, random_state=0)
     
141.     lda.fit(tf)
     
142.     plexs.append(lda.perplexity(tf))
     
143. 
     
144. n_t = 15  # 区间最右侧的值。注意：不能大于n_max_topics
     
145. x = list(range(1, n_t))
     
146. plt.plot(x, plexs[1:n_t])
     
147. plt.xlabel("number of topics")
     
148. plt.ylabel("perplexity")
     
149. plt.show()
     
150. 
     
151. 
     
152. 用的下面的数据
     
153. 
     
154. 链接：https://pan.baidu.com/s/1OkzDW_fXarDOGQg3VOYb3w
     
155. 提取码：1111
     
156. 
     
157. 问题出在jieba那里 , 谢谢
     
158. 
     
159. 
     
160. 
     
161. 
     
162. 
     
163. 
     
164. 
     
165. 
     
166. 
     
167. 
     
168. 
     
169. 
     
170. 
     
171. 
     
172. 
     
173. 
     

复制代码

isdkz

报错贴全一点，把完整的回溯过程贴上来

swanseabrian

isdkz 发表于 2022-2-12 21:41
报错贴全一点，把完整的回溯过程贴上来

1. Traceback (most recent call last):
   
2.   File "C:/Users/Administrator/PycharmProjects/pythonProject2/venv/lda/LDA20220212.py", line 67, in <module>
   
3.     data["content_cutted"] = data.content.apply(chinese_word_cut)
   
4.   File "D:\python370\lib\site-packages\pandas\core\series.py", line 4213, in apply
   
5.     mapped = lib.map_infer(values, f, convert=convert_dtype)
   
6.   File "pandas\_libs\lib.pyx", line 2403, in pandas._libs.lib.map_infer
   
7.   File "C:/Users/Administrator/PycharmProjects/pythonProject2/venv/lda/LDA20220212.py", line 46, in chinese_word_cut
   
8.     for seg_word in seg_list:
   
9.   File "D:\python370\lib\site-packages\jieba\posseg\__init__.py", line 294, in cut
   
10.     for w in dt.cut(sentence, HMM=HMM):
    
11.   File "D:\python370\lib\site-packages\jieba\posseg\__init__.py", line 249, in cut
    
12.     for w in self.__cut_internal(sentence, HMM=HMM):
    
13.   File "D:\python370\lib\site-packages\jieba\posseg\__init__.py", line 217, in __cut_internal
    
14.     sentence = strdecode(sentence)
    
15.   File "D:\python370\lib\site-packages\jieba\_compat.py", line 79, in strdecode
    
16.     sentence = sentence.decode('utf-8')
    
17. AttributeError: 'int' object has no attribute 'decode'

复制代码

就上面这些,不知道应该改哪里

isdkz

在第67行前面加上这句试试

1. data.fillna('', inplace=True)

复制代码

z5560636

1. data["content_cutted"] = data.content.apply(chinese_word_cut)

复制代码

这里错了，整形没有decode方法。

swanseabrian

isdkz 发表于 2022-2-12 22:10
在第67行前面加上这句试试

没起作用,还是报那个错误

isdkz

swanseabrian 发表于 2022-2-12 22:50
没起作用,还是报那个错误

好吧

swanseabrian

z5560636 发表于 2022-2-12 22:15
这里错了，整形没有decode方法。

那这里要怎么改一下,我不会改还,谢谢

isdkz

方便把另外两个文件放上来吗？不然不好帮你调试