python实现BERT【tensorflow2.4.0】【命名实体识别】

糖逗

一、数据预处理+模型训练

1. import os
   
2. #os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
   
3. train_data_path = "E:/.../1.NLP/地址识别项目/data/train.conll" # 训练数据
   
4. test_data_path = "E:/.../1.NLP/地址识别项目/data/dev.conll" # 测试数据
   
5. 
   
6. 
   
7. 
   
8. #创建打标的标签索引
   
9. import pandas as pd
   
10. store = pd.read_table(r"E:\...\1.NLP\地址识别项目\data\mytag.dic", header = None)
    
11. store.loc[0, 0] = "O"
    
12. store.loc[24, 0] = "B-prov"
    
13. store1 = store.to_dict()
    
14. idx2label = store1[0]
    
15. label2idx = {idx: label for label, idx in idx2label.items()}
    
16. CLASS_NUM = len(label2idx)
    
17. 
    
18. 
    
19. #读取文本数据和标注数据
    
20. from tensorflow.keras.preprocessing import sequence
    
21. import numpy as np
    
22. MAX_LEN = 55
    
23. # 读取训练语料
    
24. def read_corpus(corpus_path, label2idx):
    
25.     datas, labels = [], []
    
26.     with open(corpus_path, encoding='utf-8') as fr:
    
27.         lines = fr.readlines()
    
28.     sent_, tag_ = [], []
    
29.     for line in lines:
    
30.         if line != '\n':
    
31.             char, label = line.strip().split()
    
32.             sent_.append(char)
    
33.             tag_.append(label)
    
34.         else:
    
35.             sent_ids = [char for char in sent_]
    
36.             tag_ids = [label2idx[label] if label in label2idx else 0 for label in tag_]
    
37.             datas.append(sent_ids)
    
38.             labels.append(tag_ids)
    
39.             sent_, tag_ = [], []
    
40.     return datas, labels
    
41. 
    
42. # 加载训练集
    
43. train_datas, train_labels = read_corpus(train_data_path, label2idx)
    
44. # 加载测试集
    
45. test_datas, test_labels = read_corpus(test_data_path, label2idx)
    
46. 
    
47. # In[1]
    
48. from transformers import BertTokenizer
    
49. 
    
50. # 使用bert的tokenizer将文字转化成数字。
    
51. PRETRAINED_MODEL_NAME = r"D:\bert_model\bert-base-chinese"  # 指定为中文
    
52. tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)
    
53. 
    
54. def pad(train_datas):
    
55.     for i in range(len(train_datas)):
    
56.         #train_datas[i].insert(0, "[CLS]")
    
57.         #train_datas[i].append('[SEP]')
    
58.         if len(train_datas[i]) < MAX_LEN:
    
59.             train_datas[i].extend(['[PAD]'] * (MAX_LEN - len(train_datas[i])))
    
60.         elif len(train_datas[i]) > MAX_LEN:
    
61.             train_datas[i] = train_datas[i][0:MAX_LEN]
    
62.     return train_datas
    
63. 
    
64. train_datas_pad = np.array(pad(train_datas))
    
65. test_datas_pad = np.array(pad(test_datas))
    
66. 
    
67. def txt2bert(train_datas_pad):
    
68.     train_data_bert = []
    
69.     for i in range(len(train_datas_pad)):
    
70.         store = []
    
71.         for j in range(len(train_datas_pad[i])):
    
72.             store.append(tokenizer.convert_tokens_to_ids(train_datas_pad[i][j]))
    
73.         train_data_bert.append(store)
    
74.     return train_data_bert
    
75. 
    
76. train_data_bert = np.array(txt2bert(train_datas_pad))
    
77. test_data_bert = np.array(txt2bert(test_datas_pad))
    
78. print(train_data_bert.shape)
    
79. print(test_data_bert.shape)
    
80. print(tokenizer.convert_ids_to_tokens(train_data_bert[0]))
    
81. # In[1]
    
82. def pad_test(train_labels):
    
83.     for i in range(len(train_labels)):
    
84.         #train_labels[i].insert(0, 57)
    
85.         #train_labels[i].append(58)
    
86.         if len(train_labels[i]) < MAX_LEN:
    
87.             train_labels[i].extend([0] * (MAX_LEN - len(train_labels[i])))
    
88.         elif len(train_labels[i]) > MAX_LEN:
    
89.             train_labels[i] = train_labels[i][0:MAX_LEN]
    
90.     return train_labels
    
91.    
    
92. train_labels_pad = np.array(pad_test(train_labels))
    
93. test_labels_pad = np.array(pad_test(test_labels))
    
94. print(train_labels_pad.shape)
    
95. print(test_labels_pad.shape)
    
96. print([idx2label[char] for char in train_labels_pad[0]])
    
97. 
    
98. 
    
99. # In[3]
    
100. import tensorflow as tf
     
101. 
     
102. def preprocess(train_ids, train_labels):
     
103.     x = tf.cast(train_ids, dtype = tf.int64)
     
104.     y = tf.cast(train_labels, dtype = tf.int64)
     
105.     return x, y
     
106. 
     
107. 
     
108. 
     
109. train_db = tf.data.Dataset.from_tensor_slices((train_data_bert, train_labels_pad))
     
110. train_db = train_db.map(preprocess).batch(32)
     
111. 
     
112. 
     
113. test_db = tf.data.Dataset.from_tensor_slices((test_data_bert, test_labels_pad))
     
114. test_db = test_db.map(preprocess).batch(32)
     
115. # In[3]
     
116. from transformers import TFBertForTokenClassification
     
117. from tensorflow import keras
     
118. 
     
119. model = TFBertForTokenClassification.from_pretrained(PRETRAINED_MODEL_NAME, num_labels = CLASS_NUM)
     
120. 
     
121. 
     
122. 
     
123. model.layers[-1].activation = tf.keras.activations.softmax
     
124. 
     
125. optimizer = tf.keras.optimizers.Adam(learning_rate=1e-6)
     
126. loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
     
127. metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')
     
128. 
     
129. model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
     
130. 
     
131. model.summary()
     
132. 
     
133. 
     
134. history = model.fit(x = train_data_bert, y = train_labels_pad, validation_split=0.1,batch_size=32, epochs=10).history
     
135. 
     
136. # In[1]
     
137. 
     
138. #print(tf.test.is_gpu_available())

复制代码

1. #x_test, y_test =  next(iter(train_db))
   
2. #preds = model(input_ids = x_test, attention_mask = mask_test)
   
3. 
   
4. x_test, y_test =  next(iter(train_db))
   
5. mask_test = np.int64(x_test) >= 0
   
6. mask_test = tf.convert_to_tensor(mask_test)
   
7. preds = model(input_ids = x_test, attention_mask = mask_test)
   
8.         
   
9. prob = tf.nn.softmax(preds.logits, axis=-1)
   
10. pred = tf.argmax(prob, axis=-1)
    
11. pred = tf.cast(pred, dtype=tf.int64)
    
12. #pred = tf.reshape(pred, (1, -1))
    
13. store = tf.reduce_sum(tf.cast(x_test[1] >0 , dtype = tf.int32))
    
14. print([idx2label[char] for char in np.array(pred[1])][:int(store)])
    
15. 
    
16. print(tokenizer.convert_ids_to_tokens(x_test[1])[:int(store)])                           

复制代码

二、模型效果查看

1. #x_test, y_test =  next(iter(train_db))
   
2. #preds = model(input_ids = x_test, attention_mask = mask_test)
   
3. 
   
4. x_test, y_test =  next(iter(train_db))
   
5. mask_test = np.int64(x_test) >= 0
   
6. mask_test = tf.convert_to_tensor(mask_test)
   
7. preds = model(input_ids = x_test, attention_mask = mask_test)
   
8.         
   
9. prob = tf.nn.softmax(preds.logits, axis=-1)
   
10. pred = tf.argmax(prob, axis=-1)
    
11. pred = tf.cast(pred, dtype=tf.int64)
    
12. #pred = tf.reshape(pred, (1, -1))
    
13. store = tf.reduce_sum(tf.cast(x_test[1] >0 , dtype = tf.int32))
    
14. print([idx2label[char] for char in np.array(pred[1])][:int(store)])
    
15. 
    
16. print(tokenizer.convert_ids_to_tokens(x_test[1])[:int(store)])                           

复制代码

三、模型结果评价指标

1. test_data_bert
   
2. mask_test = np.int64(test_data_bert) >= 0
   
3. mask_test = tf.convert_to_tensor(mask_test)
   
4. preds = model(input_ids = test_data_bert, attention_mask = mask_test)
   
5. 
   
6. prob = tf.nn.softmax(preds.logits, axis = -1)
   
7. pred = tf.argmax(prob, axis = -1)
   
8. pred = tf.cast(pred, dtype = tf.int64)
   
9. 
   
10. 
    
11. 
    
12. mark_labels = test_labels_pad > 0
    
13. total_num = tf.reduce_sum(tf.cast(mark_labels, dtype = tf.int32))
    
14. print(total_num)
    
15. equal_labels = tf.equal(pred, test_labels_pad)
    
16. false_labels = test_labels_pad < 0
    
17. true_pred = tf.reduce_sum(tf.cast(tf.where(mark_labels, equal_labels, false_labels), dtype = tf.int32))
    
18. print(true_pred)
    
19. print(true_pred/total_num)
    
20. store = tf.reduce_sum(tf.cast(tf.equal(pred, test_labels_pad),  dtype=tf.int32))
    
21. print(store / (1970*55))
    
22. #x_test, y_test =  next(iter(train_db))
    
23. #preds = model(input_ids = x_test, attention_mask = mask_test)
    
24. 
    
25. x_test, y_test =  next(iter(train_db))
    
26. mask_test = np.int64(x_test) >= 0
    
27. mask_test = tf.convert_to_tensor(mask_test)
    
28. preds = model(input_ids = x_test, attention_mask = mask_test)
    
29.         
    
30. prob = tf.nn.softmax(preds.logits, axis=-1)
    
31. pred = tf.argmax(prob, axis=-1)
    
32. pred = tf.cast(pred, dtype=tf.int64)
    
33. #pred = tf.reshape(pred, (1, -1))
    
34. store = tf.reduce_sum(tf.cast(x_test[1] >0 , dtype = tf.int32))
    
35. print([idx2label[char] for char in np.array(pred[1])][:int(store)])
    
36. 
    
37. print(tokenizer.convert_ids_to_tokens(x_test[1])[:int(store)])      
    
38. pred_labels = []
    
39. mask_test = np.int64(test_data_bert) >= 0
    
40. mask_test = tf.convert_to_tensor(mask_test)
    
41. pred = model(input_ids = test_data_bert, attention_mask = mask_test)
    
42. prob = tf.nn.softmax(pred.logits, axis=-1)
    
43. pred = tf.argmax(prob, axis=-1)
    
44. pred = tf.cast(pred, dtype=tf.int64)
    
45. print(pred)
    
46. 
    
47. from sklearn.metrics import f1_score
    
48. f1_score(test_labels_pad.reshape(-1), np.array(pred).reshape(-1), average='macro')
    
49. 
    
50. from sklearn.metrics import f1_score
    
51. f1_score(test_labels_pad.reshape(-1), y_predict.reshape(-1), average='micro')
    
52. 
    
53. from sklearn.metrics import classification_report
    
54. temp = classification_report(test_labels_pad.reshape(-1), y_predict.reshape(-1))
    
55. print(temp)
    
56.                         

复制代码

糖逗

任务背景+数据获取看上一篇:https://fishc.com.cn/forum.php?m ... 05523&ctid=1732