python实现线性回归

糖逗

参考书籍：《机器学习实战》

1.传统线性回归

1. import numpy as np
   
2. 
   
3. def loadDataSet(fileName):
   
4.     numFeat = len(open(fileName).readline().split('\t')) - 1
   
5.     dataMat = []
   
6.     labelMat = []
   
7.     fr = open(fileName)
   
8.     for line in fr.readlines():
   
9.         lineArr = []
   
10.         curLine = line.strip().split('\t')
    
11.         for i in range(numFeat):
    
12.             lineArr.append(float(curLine[i]))
    
13.         dataMat.append(lineArr)
    
14.         labelMat.append(float(curLine[-1]))
    
15.     return dataMat, labelMat
    
16. 
    
17. 
    
18. def standRegres(xArr, yArr):
    
19.     xMat = np.mat(xArr)
    
20.     yMat = np.mat(yArr).T
    
21.     xTx = xMat.T * xMat
    
22.     if np.linalg.det(xTx) == 0:#矩阵不可逆,矩阵行列式为0
    
23.         print("This matrix is singular, cannot do inverse")
    
24.         return
    
25.     ws = np.linalg.inv(xTx)*(xMat.T * yMat)
    
26.     return ws
    
27. 
    
28. 
    
29. 
    
30. 
    
31. if __name__ == "__main__":
    
32.     import matplotlib.pyplot as plt
    
33. 
    
34.     xArr, yArr = loadDataSet(r'C:\...\ex0.txt')
    
35.     ws = standRegres(xArr, yArr)
    
36.     xMat = np.mat(xArr)
    
37.     yMat = np.mat(yArr)
    
38.     yHat = xMat * ws
    
39. 
    
40.     fig = plt.figure()
    
41.     ax = fig.add_subplot(111)
    
42.     #https://blog.csdn.net/lilong117194/article/details/78288795
    
43.     ax.scatter(xMat[:, 1].flatten().A[0], yMat.T[:, 0].flatten().A[0])
    
44.    
    
45.     xCopy = xMat.copy()
    
46.     xCopy.sort(0)#升序
    
47.     yHat = xCopy * ws
    
48.     ax.plot(xCopy[:, 1], yHat)
    
49.     plt.show()

复制代码

2.局部加权线性回归

1. import numpy as np
   
2. 
   
3. def loadDataSet(fileName):
   
4.     numFeat = len(open(fileName).readline().split('\t')) - 1
   
5.     dataMat = []
   
6.     labelMat = []
   
7.     fr = open(fileName)
   
8.     for line in fr.readlines():
   
9.         lineArr = []
   
10.         curLine = line.strip().split('\t')
    
11.         for i in range(numFeat):
    
12.             lineArr.append(float(curLine[i]))
    
13.         dataMat.append(lineArr)
    
14.         labelMat.append(float(curLine[-1]))
    
15.     return dataMat, labelMat
    
16. 
    
17. def lwlr(testPoint, xArr, yArr, k = 1.0):
    
18.     xMat = np.mat(xArr)
    
19.     yMat = np.mat(yArr).T
    
20.     m = np.shape(xMat)[0]
    
21.     weights = np.mat(np.eye((m)))
    
22.     for j in range(m):                     
    
23.         diffMat = testPoint - xMat[j,:]     
    
24.         weights[j,j] = np.exp(diffMat * diffMat.T/(-2.0*k**2))#径向基核函数
    
25.     xTx = xMat.T * (weights * xMat)
    
26.     if np.linalg.det(xTx) == 0.0:
    
27.         print("This matrix is singular, cannot do inverse")
    
28.         return
    
29.     ws = np.linalg.inv(xTx) * (xMat.T * weights * yMat)
    
30.     return testPoint * ws
    
31. 
    
32. def lwlrTest(testArr, xArr, yArr, k = 1.0):  
    
33.     m = np.shape(testArr)[0]
    
34.     yHat = np.zeros(m)
    
35.     for i in range(m):
    
36.         yHat[i] = lwlr(testArr[i],xArr,yArr,k)
    
37.     return yHat
    
38. 
    
39. if __name__ == "__main__":
    
40.     xArr, yArr = loadDataSet(r'C:\...\ex0.txt')
    
41.     lwlr(xArr[0], xArr, yArr, 1)#单个点的估计
    
42.     yHat = lwlrTest(xArr, xArr, yArr, 0.003)
    
43.     xMat = np.mat(xArr)
    
44.     srtInd = xMat[:, 1].argsort(0)
    
45.     xSort = xMat[srtInd][:, 0, :]
    
46.     import matplotlib.pyplot as plt
    
47.     fig = plt.figure()
    
48.     ax = fig.add_subplot(111)
    
49.     ax.plot(xSort[:, 1], yHat[srtInd])
    
50.     ax.scatter(xMat[:, 1].flatten().A[0], np.mat(yArr).T.flatten().A[0],
    
51.                s = 2, c = 'red')
    
52.     plt.show()
    

复制代码

3.岭回归

1. import numpy as np
   
2. 
   
3. def loadDataSet(fileName):
   
4.     numFeat = len(open(fileName).readline().split('\t')) - 1
   
5.     dataMat = []
   
6.     labelMat = []
   
7.     fr = open(fileName)
   
8.     for line in fr.readlines():
   
9.         lineArr = []
   
10.         curLine = line.strip().split('\t')
    
11.         for i in range(numFeat):
    
12.             lineArr.append(float(curLine[i]))
    
13.         dataMat.append(lineArr)
    
14.         labelMat.append(float(curLine[-1]))
    
15.     return dataMat, labelMat
    
16. 
    
17. def ridgeRegres(xMat, yMat, lam = 0.2):
    
18.     xTx = xMat.T * xMat
    
19.     denom = xTx + np.eye(np.shape(xMat)[1]) * lam
    
20.     if np.linalg.det(denom) == 0:
    
21.         print("This matrix is singular, cannot do inverse")
    
22.         return
    
23.     ws = np.linalg.inv(denom) * (xMat.T * yMat)
    
24.     return ws
    
25. 
    
26. def ridgeTest(xArr, yArr):
    
27.     xMat = np.mat(xArr)
    
28.     yMat = np.mat(yArr).T
    
29.     yMean = np.mean(yMat,0)
    
30.     yMat = yMat - yMean   
    
31.     xMeans = np.mean(xMat,0)
    
32.     xVar = np.var(xMat,0)      
    
33.     xMat = (xMat - xMeans)/xVar
    
34.     numTestPts = 30#30个不同的lambda
    
35.     wMat = np.zeros((numTestPts, np.shape(xMat)[1]))
    
36.     for i in range(numTestPts):
    
37.         ws = ridgeRegres(xMat, yMat, np.exp(i-10))
    
38.         wMat[i, :] = ws.T
    
39.     return wMat
    
40. 
    
41. if __name__ == "__main__":
    
42.     abX, abY = loadDataSet(r'C:\...\abalone.txt')
    
43.     ridgeWeights = ridgeTest(abX, abY)
    
44.     import matplotlib.pyplot as plt
    
45.     fig = plt.figure()
    
46.     ax = fig.add_subplot(111)
    
47.     ax.plot(ridgeWeights）#一共有八条线，代表不同lambda下8个特征的取值情况
    
48.     plt.show()

复制代码

4.前向逐步回归

1. import numpy as np
   
2. 
   
3. def loadDataSet(fileName):
   
4.     numFeat = len(open(fileName).readline().split('\t')) - 1
   
5.     dataMat = []
   
6.     labelMat = []
   
7.     fr = open(fileName)
   
8.     for line in fr.readlines():
   
9.         lineArr = []
   
10.         curLine = line.strip().split('\t')
    
11.         for i in range(numFeat):
    
12.             lineArr.append(float(curLine[i]))
    
13.         dataMat.append(lineArr)
    
14.         labelMat.append(float(curLine[-1]))
    
15.     return dataMat, labelMat
    
16. 
    
17. def regularize(xMat):
    
18.     inMat = xMat.copy()
    
19.     inMeans = np.mean(inMat,0)   
    
20.     inVar = np.var(inMat,0)     
    
21.     inMat = (inMat - inMeans)/inVar
    
22.     return inMat
    
23. 
    
24. def rssError(yArr, yHatArr):
    
25.     return ((yArr-yHatArr)**2).sum()
    
26. 
    
27. def stageWise(xArr, yArr, eps = 0.01, numIt = 100):
    
28.     xMat = np.mat(xArr)
    
29.     yMat = np.mat(yArr).T
    
30.     yMean = np.mean(yMat, 0)
    
31.     yMat = yMat - yMean     
    
32.     xMat = regularize(xMat)
    
33.     m, n = np.shape(xMat)
    
34.     ws = np.zeros((n, 1))
    
35.     returnMat = np.zeros((numIt, n))
    
36.     wsMax = ws.copy()
    
37.     for i in range(numIt):
    
38.         lowestError = np.inf;
    
39.         for j in range(n):
    
40.             for sign in [-1, 1]:#增大或减少特征值
    
41.                 wsTest = ws.copy()
    
42.                 wsTest[j] += eps * sign
    
43.                 yTest = xMat * wsTest
    
44.                 rssE = rssError(yMat.A,yTest.A)
    
45.                 if rssE < lowestError:
    
46.                     lowestError = rssE
    
47.                     wsMax = wsTest
    
48.         ws = wsMax.copy()
    
49.         returnMat[i,:] = ws.T
    
50.     return returnMat
    
51. 
    
52. if __name__ == "__main__":
    
53.     xArr, yArr = loadDataSet(r'C:\...\abalone.txt')
    
54.     res = stageWise(xArr, yArr, 0.01, 5000)
    
55.     import matplotlib.pyplot as plt
    
56.     fig = plt.figure()
    
57.     ax = fig.add_subplot(111)
    
58.     ax.plot(res)#一共有八条线，代表不同lambda下8个特征的取值情况
    
59.     plt.show()

复制代码

糖逗

糖逗

补充一个知识点：python中x.I就是对矩阵x求逆矩阵

糖逗

基于批梯度参数更新的线性回归

1. import numpy as np
   
2. import matplotlib.pyplot as plt
   
3. #计算损失函数
   
4. def compute_error_for_line_given_points(b, w, points):
   
5.     totalError = 0
   
6.     for i in range(0, len(points)):
   
7.         x = points[i, 0]
   
8.         y = points[i, 1]
   
9.         totalError += (y - (w * x + b)) ** 2
   
10.     return totalError / len(points)
    
11. 
    
12. #计算梯度(批梯度下降，计算一次梯度使用所有样本)，更新参数
    
13. #一次参数更新计算
    
14. def step_gradient(b_current, w_current, points, learningRate):
    
15.     b_gradient = 0
    
16.     w_gradient = 0
    
17.     N = len(points)
    
18.     for i in range(N):
    
19.         x = points[i, 0]
    
20.         y = points[i, 1]
    
21.         #grad_b = 2(wx + b - y)
    
22.         b_gradient += (2 / N) * ((w_current * x + b_current) - y)
    
23.         #grad_w = 2(wx+b-y)*x
    
24.         w_gradient += (2/ N) * x * ((w_current * x + b_current) - y)
    
25.     new_b = b_current - (learningRate * b_gradient)
    
26.     new_w = w_current - (learningRate * w_gradient)
    
27.     return [new_b, new_w]
    
28. 
    
29. #多次梯度下降计算迭代
    
30. def gradient_descent_runner(points, starting_b, starting_w, learning_rate, num_iterations):
    
31.     b = starting_b
    
32.     w = starting_w
    
33.     for i in range(num_iterations):
    
34.         b, w = step_gradient(b, w, np.array(points), learning_rate)
    
35.     return [b, w]
    
36. 
    
37. 
    
38. if __name__ == "__main__":
    
39.     points = [[6, 24], [2, 80], [9, 2], [1, 100], [2, 98], [10, 12], [4, 77], [4.5, 60],[6.5, 15]]
    
40.     plt.scatter(x = np.array(points)[:,0] , y = np.array(points)[:,1])
    
41.     b, w = gradient_descent_runner(points, 100, -10, 0.02, 500)#初始化参数选取很重要，尤其是学习率
    
42.     x = np.random.rand(100) * 10
    
43.     y = np.array([])
    
44.     for i in range(len(x)):
    
45.         y = np.append(y, w * x[i] + b)
    
46.     plt.plot(x, y)

复制代码

代码源自：《Tensorflow深度学习》（龙良曲）