解读自定义的神经网络模型

Terence888

1. class Net(torch.nn.Module):
   
2.     def __init__(self):
   
3.         super(Net,self).__init__()
   
4.         self.hidden_layer1 = nn.Linear(331,1024)
   
5.         self.hidden_layer2 = nn.Linear(1024,1024)
   
6.         self.hidden_layer3 = nn.Linear(1024,1024)
   
7.         self.hidden_layer4 = nn.Linear(1024,1024)
   
8.         self.output_layer = nn.Linear(1024,1)
   
9.         self.dropout = nn.Dropout(p=0.2)
   
10.         nn.init.xavier_uniform_(self.hidden_layer1.weight)
    
11.         nn.init.xavier_uniform_(self.hidden_layer2.weight)
    
12.         nn.init.xavier_uniform_(self.hidden_layer3.weight)
    
13.         nn.init.xavier_uniform_(self.hidden_layer4.weight)
    
14.         nn.init.xavier_uniform_(self.output_layer.weight)
    
15.     def forward(self,x):
    
16.         inputs = x
    
17.         layer1_out = torch.nn.functional.gelu(self.hidden_layer1(inputs))
    
18.         layer1_out = self.dropout(layer1_out)
    
19.         layer2_out = torch.nn.functional.gelu(self.hidden_layer2(layer1_out))
    
20.         layer2_out = self.dropout(layer2_out)
    
21.         layer3_out = torch.nn.functional.gelu(self.hidden_layer3(layer2_out))
    
22.         layer3_out = self.dropout(layer3_out)
    
23.         layer4_out = torch.nn.functional.gelu(self.hidden_layer4(layer3_out))
    
24.         layer4_out = self.dropout(layer4_out)
    
25.         output = torch.relu(self.output_layer(layer4_out))
    
26.         return output

复制代码

这个网络的gelu函数是什么？nn.init.xavier_uniform_是什么方法？

FishC

这个自定义的神经网络模型有几个关键点：

1. **GELU函数**：GELU（Gaussian Error Linear Unit）是激活函数，常用于神经网络中。它结合了ReLU和Sigmoid的优点，通过保留输入的高斯分布特性，以提升模型性能，公式为：
   \[
   \text{GELU}(x) = x \cdot \Phi(x)
   \]
   其中，\(\Phi(x)\) 是标准正态分布的累积分布函数。

2. **Xavier初始化**：`nn.init.xavier_uniform_`是用来初始化网络权重的方法。通过维持每层输出的方差一致，能有效减轻梯度消失或爆炸问题。Xavier均匀初始化在输入和输出神经元数量的基础上，使用一个均匀分布来初始化权重。

这个模型使用了多层线性层和GELU激活函数，并加入了Dropout正则化，以减少过拟合。最终输出用ReLU激活函数处理。

以上回复来自 FishC AI 小助理，如未能正确解答您的问题，请继续追问。

Terence888

FishC 发表于 2024-10-26 18:33
这个自定义的神经网络模型有几个关键点：

1. **GELU函数**：GELU（Gaussian Error Linear Unit）是激活 ...

归一化之前损失特别高，我对特征进行了归一化，损失一下子降到零点几，是什么原因

FishC

Terence888 发表于 2024-10-26 19:01
归一化之前损失特别高，我对特征进行了归一化，损失一 ...

特征归一化能够帮助模型更快收敛，主要原因如下：

1. **数值稳定性**：归一化后的特征值通常更小且在相似的范围内，这可以增加数值计算的稳定性。

2. **梯度下降效率**：特征值范围一致，使得梯度下降算法能够更有效地调整权重，避免某些特征主导梯度更新。

3. **避免偏置问题**：未归一化的特征可能导致模型偏向处理某些特征，归一化能够防止这种情况。

因此，通过归一化，模型可以更好地训练和收敛，从而明显降低损失。

以上回复来自 FishC AI 小助理，如未能正确解答您的问题，请继续追问。

Terence888

Terence888 发表于 2024-10-26 19:01
归一化之前损失特别高，我对特征进行了归一化，损失一下子降到零点几，是什么原因

1. 
   
2. #使用dnn模型（k折交叉验证）
   
3. import torch
   
4. import torch.nn as nn
   
5. from torch.utils import data
   
6. from torch.utils.data import Dataset,DataLoader
   
7. from torch import optim
   
8. 
   
9. #定义神经网络模型
   
10. 
    
11. class SimpleNN(nn.Module):
    
12.     def __init__(self):
    
13.         super(SimpleNN, self).__init__()  # 继承需要用 SimpleNN
    
14.         self.hidden_layer1 = nn.Linear(154,1024)
    
15.         self.hidden_layer2 = nn.Linear(1024,1024)
    
16.         self.hidden_layer3 = nn.Linear(1024,1024)
    
17.         self.hidden_layer4 = nn.Linear(1024,1024)
    
18.         self.output_layer = nn.Linear(1024,1)
    
19.         self.dropout = nn.Dropout(0.2)
    
20.         nn.init.xavier_uniform_(self.hidden_layer1.weight) #初始化每层网络的权重参数
    
21.         nn.init.xavier_uniform_(self.hidden_layer2.weight)
    
22.         nn.init.xavier_uniform_(self.hidden_layer3.weight)
    
23.         nn.init.xavier_uniform_(self.hidden_layer4.weight)
    
24.         nn.init.xavier_uniform_(self.output_layer.weight)
    
25.       
    
26.     def forward(self, X):
    
27.         inputs = X
    
28.         layer1_out = torch.nn.functional.gelu(self.hidden_layer1(inputs))
    
29.         layer1_out = self.dropout(layer1_out)
    
30.         layer2_out = torch.nn.functional.gelu(self.hidden_layer2(layer1_out))
    
31.         layer2_out = self.dropout(layer2_out)
    
32.         layer3_out = torch.nn.functional.gelu(self.hidden_layer3(layer2_out))
    
33.         layer3_out = self.dropout(layer3_out)
    
34.         layer4_out = torch.nn.functional.gelu(self.hidden_layer4(layer3_out))
    
35.         layer4_out = self.dropout(layer4_out)
    
36.         output = torch.relu(self.output_layer(layer4_out))
    
37.         return output
    
38.    
    
39. #初始化模型和优化器
    
40. dnn_model = SimpleNN()
    
41. loss = nn.MSELoss() #定义损失函数
    
42. optimizer = optim.Adam(dnn_model.parameters(),lr=0.0001,weight_decay=0) #定义优化器
    
43. 
    
44. 
    
45. #k折交叉验证选取训练集与验证集
    
46. def get_k_fold_data(k, i, X, y):
    
47.     assert k > 1
    
48.     fold_size = len(X) // k
    
49.     X_train, y_train = None, None
    
50.     for j in range(k):
    
51.         start = j * fold_size
    
52.         end = (j + 1) * fold_size
    
53.         if j == i:
    
54.             X_valid, y_valid = X.iloc[start:end], y.iloc[start:end]
    
55.         elif X_train is None:
    
56.             X_train, y_train = X.iloc[start:end], y.iloc[start:end]
    
57.         else:
    
58.             X_train = pd.concat([X_train, X.iloc[start:end]], ignore_index=True)
    
59.             y_train = pd.concat([y_train, y.iloc[start:end]], ignore_index=True)
    
60.     return X_train, y_train, X_valid, y_valid
    
61. 
    
62. 
    
63. # 开始训练
    
64. k = 5
    
65. batch_size = 64
    
66. num_epochs = 100
    
67. #weight_decay = 0
    
68. 
    
69. #初始化损失
    
70. train_l_sum, valid_l_sum = 0, 0
    
71. 
    
72. #初始化列表
    
73. train_ls, valid_ls = [], []
    
74. 
    
75. for i in range(k):
    
76.     X_train, y_train, X_valid, y_valid = get_k_fold_data(k, i, X, y)
    
77.     print(f'FOLD {i}')
    
78.     print('--------------------------------')
    
79.    
    
80. 
    
81.     #将DataFrame数据转换为NumPy数组，然后再转换为PyTorch张量
    
82.     X_train = torch.tensor(X_train.astype(np.float32).values, dtype=torch.float32)
    
83.     y_train = torch.tensor(y_train.astype(np.float32).values, dtype=torch.float32)
    
84.     X_valid = torch.tensor(X_valid.astype(np.float32).values, dtype=torch.float32)
    
85.     y_valid = torch.tensor(y_valid.astype(np.float32).values, dtype=torch.float32)
    
86.    
    
87.     #创建数据集
    
88.     train_dataset = data.TensorDataset(X_train, y_train)
    
89.     valid_dataset = data.TensorDataset(X_valid, y_valid)
    
90. 
    
91.     # 获取一个数据迭代器
    
92.     train_iter = DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True,num_workers=2)#shuffle=True相当于sampler=RandomSampler(dataset)
    
93.     valid_iter = DataLoader(dataset=valid_dataset,batch_size=batch_size,shuffle=True,num_workers=2)
    
94.    
    
95.     #开始迭代
    
96.     for epoch in range(num_epochs):
    
97.         train_loss = 0
    
98.         for tensor_x, tensor_y in train_iter:#训练集执行梯度更新
    
99.             tensor_x = tensor_x.float()
    
100.             tensor_y = tensor_y.float().reshape(-1, 1)
     
101.             optimizer.zero_grad() #梯度清零
     
102.             pre_train = dnn_model(tensor_x)
     
103.             train_l = loss(pre_train, tensor_y) #损失应避免与全局变量loss重名
     
104.             train_l.backward()#前向传播
     
105.             optimizer.step()#梯度下降
     
106. 
     
107.             train_loss += train_l.item() * len(tensor_x) #批量规模损失累加
     
108.             
     
109.         train_loss /= len(train_dataset) #每次迭代平均损失
     
110.         
     
111.         if epoch % 20 == 0:
     
112.             print('Loss: {}  Epoch:{}'.format(train_loss, epoch))
     
113.         
     
114.         with torch.no_grad():
     
115.             valid_loss = 0
     
116.             
     
117.             for tensor_x, tensor_y in valid_iter:
     
118.                 tensor_x = tensor_x.float()
     
119.                 tensor_y = tensor_y.float().reshape(-1, 1)
     
120.                 pre_valid = dnn_model(tensor_x)
     
121.                 valid_l = loss(pre_valid, tensor_y)
     
122.                 valid_loss += valid_l.item() * len(tensor_x)
     
123.                   
     
124.             valid_loss /= len(valid_dataset)
     
125.                   
     
126.             if epoch % 20 == 0:
     
127.                 print('Valid Loss: {}  Epoch:{}'.format(valid_loss, epoch))
     
128.         
     
129.     if i == 0:
     
130.         plot(list(range(1, num_epochs + 1)), [train_ls, valid_ls],xlabel='epoch', ylabel='rmse', xlim=[1, num_epochs],legend=['train', 'valid'], yscale='log')
     
131.     #将每折的损失添加到列表中
     
132.     train_ls.append(train_loss)
     
133.     valid_ls.append(valid_loss)
     
134. 
     
135.         
     
136. print('Training Ended')
     
137. print('Train Average Loss: {} Valid Average Loss: {}'.format(np.mean(train_ls),np.mean(valid_ls)))

复制代码

自定义的神经网络如上，结果如下，代码存在什么问题？

1. FOLD 0
   
2. --------------------------------
   
3. Loss: 36344742366.57045  Epoch:0
   
4. Valid Loss: 29782921659.381443  Epoch:0
   
5. Loss: 3065413660.5910654  Epoch:20
   
6. Valid Loss: 2382111456.329897  Epoch:20
   
7. Loss: 2432799436.09622  Epoch:40
   
8. Valid Loss: 2090321211.8213058  Epoch:40
   
9. Loss: 2391088845.85567  Epoch:60
   
10. Valid Loss: 1990942727.9175258  Epoch:60
    
11. Loss: 2284166063.065292  Epoch:80
    
12. Valid Loss: 2163259427.1890035  Epoch:80
    
13. FOLD 1
    
14. --------------------------------
    
15. Loss: 2206484537.182131  Epoch:0
    
16. Valid Loss: 2204314296.742268  Epoch:0
    
17. Loss: 1998103985.2646048  Epoch:20
    
18. Valid Loss: 2287907708.9209623  Epoch:20
    
19. Loss: 2017798130.1443298  Epoch:40
    
20. Valid Loss: 2118876600.3024056  Epoch:40
    
21. Loss: 1974789904.274914  Epoch:60
    
22. Valid Loss: 2002172580.5085912  Epoch:60
    
23. Loss: 1853698483.4639175  Epoch:80
    
24. Valid Loss: 1899964549.718213  Epoch:80
    
25. FOLD 2
    
26. --------------------------------
    
27. Loss: 1769865234.5841925  Epoch:0
    
28. Valid Loss: 1894314067.573883  Epoch:0
    
29. Loss: 1821377592.742268  Epoch:20
    
30. Valid Loss: 2032567193.95189  Epoch:20
    
31. Loss: 1743828709.6082475  Epoch:40
    
32. Valid Loss: 2015808563.024055  Epoch:40
    
33. Loss: 1722286362.8316152  Epoch:60
    
34. Valid Loss: 1898440543.0103092  Epoch:60
    
35. Loss: 1575469064.7972507  Epoch:80
    
36. Valid Loss: 1905074117.9381444  Epoch:80
    
37. FOLD 3
    
38. --------------------------------
    
39. Loss: 1642522463.8900344  Epoch:0
    
40. Valid Loss: 1701700081.4845362  Epoch:0
    
41. Loss: 1600776652.975945  Epoch:20
    
42. Valid Loss: 1750697497.5120275  Epoch:20
    
43. Loss: 1523211206.1580756  Epoch:40
    
44. Valid Loss: 1828443650.8591065  Epoch:40
    
45. Loss: 1462216860.5910654  Epoch:60
    
46. Valid Loss: 1648079266.3092782  Epoch:60
    
47. Loss: 1537743156.5635738  Epoch:80
    
48. Valid Loss: 1658678443.1065292  Epoch:80
    
49. FOLD 4
    
50. --------------------------------
    
51. Loss: 1444227294.790378  Epoch:0
    
52. Valid Loss: 1351599504.274914  Epoch:0
    
53. Loss: 1354902019.079038  Epoch:20
    
54. Valid Loss: 1279266340.7285223  Epoch:20
    
55. Loss: 1205427271.257732  Epoch:40
    
56. Valid Loss: 1247919564.5360825  Epoch:40
    
57. Loss: 1265103737.8419244  Epoch:60
    
58. Valid Loss: 1256292511.0103092  Epoch:60
    
59. Loss: 1193805096.467354  Epoch:80
    
60. Valid Loss: 1190893154.969072  Epoch:80
    
61. Training Ended
    
62. Train Average Loss: 1595598623.2302403 Valid Average Loss: 1713117385.5450172

复制代码