线性回归

# 基于numpy实现线性回归模型

import numpy as np
import pandas as pd
from sklearn.datasets import load_diabetes
from sklearn.utils import shuffle


# 初始化模型参数
def initialize_params(dims):
    w = np.zeros((dims, 1))  # 初始化权重参数为零矩阵
    b = 0  # 初始化偏差参数为零
    return w, b


# 定义模型主体部分，包括线性回归公式、均方损失和参数偏导三部分
def linear_loss(X, y, w, b):
    num_train = X.shape[0]  # 训练样本数量
    num_feature = X.shape[1]  # 训练特征数量
    y_hat = np.dot(X, w) + b  # 线性回归预测输出
    loss = np.sum((y_hat - y) ** 2) / num_train  # 计算预测输出与实际标签之间的均方损失
    dw = np.dot(X.T, (y_hat - y)) / num_train  # 基于均方损失对权重参数的一阶偏导数
    db = np.sum((y_hat - y)) / num_train  # 基于均方损失对偏差项的一阶偏导数
    return y_hat, loss, dw, db


# 定义线性回归模型训练过程
def linear_train(X, y, learning_rate=0.01, epochs=10000):
    loss_his = []  # 记录训练损失的空列表
    w, b = initialize_params(X.shape[1])  # 初始化模型参数
    for i in range(1, epochs):  # 迭代训练
        # 计算当前迭代的预测值、损失和梯度
        y_hat, loss, dw, db = linear_loss(X, y, w, b)
        # 基于梯度下降的参数更新
        w += -learning_rate * dw
        b += -learning_rate * db
        loss_his.append(loss)  # 记录当前迭代的损失
        if i % 10000 == 0:  # 每1000次迭代打印当前损失信息
            print('epoch %d loss %f' % (i, loss))
        # 将当前迭代步优化后的参数保存到字典
        params = {
            'w': w,
            'b': b
        }
        # 将当前迭代步的梯度保存到字典
        grads = {
            'dw': dw,
            'db': db
        }
    return loss_his, params, grads


# 从scikit-learn中读取数据
diabetes = load_diabetes()  # 获取diabetes数据集
data, target = diabetes.data, diabetes.target  # 获取输入和标签
X, y = shuffle(data, target, random_state=13)  # 打乱数据集
offset = int(X.shape[0] * 0.8)  # 按照8/2划分训练集和测试集
X_train, y_train = X[:offset], y[:offset]  # 训练集
X_test, y_test = X[offset:], y[offset:]  # 测试集
y_train = y_train.reshape((-1, 1))  # 将训练集改为列向量的形式
y_test = y_test.reshape((-1, 1))  # 将验证集改为列向量的形式
# 打印训练集和测试集维度
print("X_train's shape: ", X_train.shape)
print("X_test's shape: ", X_test.shape)
print("y_train's shape: ", y_train.shape)
print("y_test's shape: ", y_test.shape)

# 线性回归模型训练
loss_his, params, grads = linear_train(X_train, y_train, 0.01, 200000)
print(params)  # 打印训练后得到模型参数


# 定义线性回归预测函数
def predict(X, params):
    w = params['w']
    b = params['b']
    y_pred = np.dot(X, w) + b
    return y_pred


y_pred = predict(X_test, params)
print(y_pred[:5])


# 定义R2函数
def r2_score(y_test, y_pred):
    y_avg = np.mean(y_test)  # 测试标签均值
    ss_tot = np.sum((y_test - y_avg) ** 2)  # 总离差平方和
    ss_res = np.sum((y_test - y_pred) ** 2)  # 残差平方和
    r2 = 1 - (ss_res / ss_tot)  # R2计算
    return r2


print(r2_score(y_test, y_pred))

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_boston
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import PolynomialFeatures
from sklearn.metrics import r2_score
from sklearn.decomposition import PCA

boston_data = load_boston()  # 载入波士顿房价数据

X = boston_data['data']
y = boston_data['target']
feature_names = boston_data['feature_names']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)  # 将数据划分为训练集和测试集

print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)  # 共506个样本，每个样本有13个特征
print(feature_names)  # ['CRIM' 'ZN' 'INDUS' 'CHAS' 'NOX' 'RM' 'AGE' 'DIS', ...]

# 线性回归模型
lin_reg = LinearRegression()  # 定义线性回归对象
lin_reg.fit(X_train, y_train)  # 训练
y_pred = lin_reg.predict(X_test)  # 测试
print('r2 score in test set using 13 original features: ', r2_score(y_test, y_pred))  # R2 score约为67.7

# 为原始数据添加多项式特征
poly_feature = PolynomialFeatures(degree=2)  # 当degree>2时，出现过拟合现象

X_train_poly = poly_feature.fit_transform(X_train)
X_test_poly = poly_feature.fit_transform(X_test)
print(X_train_poly.shape, X_test_poly.shape)

lin_reg_poly = LinearRegression()
lin_reg_poly.fit(X_train_poly, y_train)
y_pred = lin_reg_poly.predict(X_test_poly)
print('r2 score in test set using polynomial features: ', r2_score(y_test, y_pred))  # R2 score约为81.8

# 仅选择一个特征(5)对数据和模型进行可视化
line_reg_one = LinearRegression()
line_reg_one.fit(X_train[:, 5].reshape(-1, 1), y_train)
y_pred = line_reg_one.predict(X_test[:, 5].reshape(-1, 1))

# 绘制数据分布
plt.scatter(X_train[:, 5], y_train, label='training set')
plt.scatter(X_test[:, 5], y_test, label='test set')

# 绘制训练得到的模型
x_plot = np.linspace(3, 9, 100)
y_plot = line_reg_one.predict(x_plot.reshape(-1, 1))
plt.plot(x_plot, y_plot, color='red')
plt.title('linear regression with one feature')
plt.legend()
plt.show()

# 欠拟合、拟合和过拟合在线性回归中的的示例
X_fit = np.linspace(-2, 2, 100)
y_fit = X_fit * X_fit + np.random.normal(0, 0.5, 100)

lin_reg_fit = LinearRegression()

poly_list = [1, 2, 50]
for i in range(len(poly_list)):
    poly_feature_fit = PolynomialFeatures(degree=poly_list[i])
    X_poly_fit = poly_feature_fit.fit_transform(X_fit.reshape(-1, 1))
    print(poly_list[i], X_poly_fit.shape)

    lin_reg_fit.fit(X_poly_fit, y_fit)
    y_pred_fit = lin_reg_fit.predict(poly_feature_fit.fit_transform(X_fit.reshape(-1, 1)))
    plt.plot(X_fit, y_pred_fit, label='degree: {}'.format(poly_list[i]))

plt.scatter(X_fit, y_fit)
plt.title('under-fitting, fitting and over-fitting')
plt.legend()
plt.show()