逻辑回归与最大熵模型

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score, classification_report


# 定义sigmoid函数
def sigmoid(x):
    z = 1 / (1 + np.exp(-x))
    return z


# 定义参数初始化函数
def initialize_params(dims):
    W = np.zeros((dims, 1))
    b = 0
    return W, b


# 定义对数几率回归模型主体
def logistic(X, y, W, b):
    num_train = X.shape[0]  # 训练样本量
    num_feature = X.shape[1]  # 训练特征数
    a = sigmoid(np.dot(X, W) + b)  # 逻辑回归模型输出
    cost = -1 / num_train * np.sum(y * np.log(a) + (1 - y) * np.log(1 - a))  # 交叉熵损失
    dW = np.dot(X.T, (a - y)) / num_train  # 权值梯度
    db = np.sum(a - y) / num_train  # 偏置梯度
    cost = np.squeeze(cost)  # 压缩损失数组维度
    return a, cost, dW, db


# 定义对数几率回归模型训练过程
def logistic_train(X, y, learning_rate, epochs):
    W, b = initialize_params(X.shape[1])  # 初始化模型参数
    cost_list = []  # 初始化损失列表
    for i in range(epochs):  # 迭代训练
        a, cost, dW, db = logistic(X, y, W, b)
        # 参数更新
        W = W - learning_rate * dW
        b = b - learning_rate * db
        # 记录损失
        if i % 100 == 0:
            cost_list.append(cost)
        if i % 100 == 0:
            print('epoch %d cost %f' % (i, cost))
    params = {'W': W, 'b': b}  # 保存参数
    grads = {'dW': dW, 'db': db}  # 保存梯度
    return cost_list, params, grads


# 定义预测函数
def predict(X, params):
    # 模型预测值
    y_prediction = sigmoid(np.dot(X, params['W']) + params['b'])
    # 基于分类阈值对概率预测值进行类别转换
    for i in range(len(y_prediction)):
        if y_prediction[i] > 0.5:
            y_prediction[i] = 1
        else:
            y_prediction[i] = 0

    return y_prediction


# 生成100*2的模拟二分类数据集
X, labels = make_classification(
    n_samples=100,
    n_features=2,
    n_redundant=0,
    n_informative=2,
    random_state=1,
    n_clusters_per_class=2)

# 设置随机数种子
rng = np.random.RandomState(2)
# 对生成的特征数据添加一组均匀分布噪声
X += 2 * rng.uniform(size=X.shape)
# 标签类别数
unique_lables = set(labels)
# 根据标签类别数设置颜色
colors = plt.cm.Spectral(np.linspace(0, 1, len(unique_lables)))
# 绘制模拟数据的散点图
for k, col in zip(unique_lables, colors):
    x_k = X[labels == k]
    plt.plot(x_k[:, 0], x_k[:, 1], 'o', markerfacecolor=col, markeredgecolor="k",
             markersize=14)
plt.title('Simulated binary data set')
plt.show()

print(X.shape, labels.shape)
labels = labels.reshape((-1, 1))
data = np.concatenate((X, labels), axis=1)
print(data.shape)
# 训练集与测试集的简单划分
offset = int(X.shape[0] * 0.9)
X_train, y_train = X[:offset], labels[:offset]
X_test, y_test = X[offset:], labels[offset:]
y_train = y_train.reshape((-1, 1))
y_test = y_test.reshape((-1, 1))

print('X_train=', X_train.shape)
print('X_test=', X_test.shape)
print('y_train=', y_train.shape)
print('y_test=', y_test.shape)

cost_list, params, grads = logistic_train(X_train, y_train, 0.01, 1000)
y_pred = predict(X_test, params)
print(classification_report(y_test, y_pred))

import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler


def plot_decision_boundary(model, axis):
    """
    在axis范围内绘制模型model的决策边界
    :param model: classification model which must have 'predict' function
    :param axis: [left, right, down, up]
    """
    x0, x1 = np.meshgrid(
        np.linspace(axis[0], axis[1], int((axis[1] - axis[0]) * 100)).reshape(-1, 1),
        np.linspace(axis[2], axis[3], int((axis[3] - axis[2]) * 100)).reshape(-1, 1),
    )
    X_new = np.c_[x0.ravel(), x1.ravel()]

    y_predict = model.predict(X_new)
    zz = y_predict.reshape(x0.shape)

    from matplotlib.colors import ListedColormap
    custom_cmap = ListedColormap(['#EF9A9A', '#FFF59D', '#90CAF9'])

    plt.contourf(x0, x1, zz, linewidth=5, cmap=custom_cmap)


# 使用numpy库构建自定义数据集
np.random.seed(666)
X = np.random.normal(0, 1, size=(200, 2))
y = np.array((X[:, 0] ** 2 + X[:, 1]) < 1.5, dtype='int')
for _ in range(20):
    y[np.random.randint(200)] = 1

plt.scatter(X[y == 0, 0], X[y == 0, 1])
plt.scatter(X[y == 1, 0], X[y == 1, 1])
plt.show()

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)

log_reg = LogisticRegression()  # 定义逻辑回归类的对象
log_reg.fit(X_train, y_train)  # 训练

print('classification accuracy of original logistic regression: ', log_reg.score(X_test, y_test))  # 评分函数

# 绘制原始逻辑回归模型的决策边界
plot_decision_boundary(log_reg, axis=[-4, 4, -4, 4])
plt.scatter(X[y == 0, 0], X[y == 0, 1])
plt.scatter(X[y == 1, 0], X[y == 1, 1])
plt.show()


# 使用Pipeline添加多项式特征、归一化后再应用逻辑回归算法
def PolynomialLogisticRegression(degree):
    return Pipeline([
        ('poly', PolynomialFeatures(degree=degree)),
        ('std_scaler', StandardScaler()),
        ('log_reg', LogisticRegression())
    ])


poly_log_reg = PolynomialLogisticRegression(degree=2)
poly_log_reg.fit(X_train, y_train)

print('classification accuracy of polynomial logistic regression: ', poly_log_reg.score(X_test, y_test))

# 绘制添加了多项式特征后的逻辑回归算法的决策边界
plot_decision_boundary(poly_log_reg, axis=[-4, 4, -4, 4])
plt.scatter(X[y == 0, 0], X[y == 0, 1])
plt.scatter(X[y == 1, 0], X[y == 1, 1])
plt.show()