softmax回归的实现

softmax回归是logistic回归在多分类问题上的推广

原理

网络架构：
在这里插入图片描述

常用的方式是独热编码：

如果下面这样，会使得分类器更倾向于把奶牛和耗牛预测到一起，因为预测为海公牛惩罚更大，这样是不合理的。

损失函数：交叉熵损失（这个函数的由来是最小化负对数似然）
在这里插入图片描述
如果预测奶牛的概率是1，则log1=0，损失为0，这是我们想要的。也就是，正确概率越大，损失越小。推广到多个样本：

将模型公式代入损失函数，得到关于w,b的损失函数：

然后使用梯度下降法，梯度是：
在这里插入图片描述

手动实现

这个代码可以在vscode而不是jupyter notebook中运行。

import torch 
import torchvision
from torch.utils import data
from torchvision import transforms
import matplotlib.pyplot as plt
#加载fashion_mnist数据集
def load_data_fashion_mnist(batch_size, resize=None):"""下载Fashion-MNIST数据集，然后将其加载到内存中"""trans = [transforms.ToTensor()]if resize:trans.insert(0, transforms.Resize(resize))trans = transforms.Compose(trans)mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)#print(len(mnist_train),len(mnist_test))return (data.DataLoader(mnist_train, batch_size, shuffle=True),data.DataLoader(mnist_test, batch_size, shuffle=False)) #windows下不能多进程,linux下可以#labels数字转为文字标签？
def get_fashion_mnist_labels(labels):text_labels=['t-shirt', 'trouser', 'pullover', 'dress', 'coat','sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']return [text_labels[int(i)] for i in labels]
#展示数据集图片的函数
def show_images(imgs, num_rows, num_cols, titles=None, scale=1.5): """绘制图像列表"""figsize = (num_cols * scale, num_rows * scale)_, axes = plt.subplots(num_rows, num_cols, figsize=figsize)axes = axes.flatten()for i, (ax, img) in enumerate(zip(axes, imgs)):if torch.is_tensor(img):# 图⽚张量ax.imshow(img.numpy())else:# PIL图⽚ax.imshow(img)ax.axes.get_xaxis().set_visible(False)ax.axes.get_yaxis().set_visible(False)if titles:ax.set_title(titles[i])plt.show()return axes#展示数据集，没什么用
# trans = transforms.ToTensor()
# mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)
# X, y = next(iter(data.DataLoader(mnist_train, batch_size=18)))
# show_images(X.reshape(18, 28, 28), 2, 9, titles=get_fashion_mnist_labels(y))#用于绘图，设置轴
def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):"""设置matplotlib的轴"""axes.set_xlabel(xlabel)axes.set_ylabel(ylabel)axes.set_xscale(xscale)axes.set_yscale(yscale)axes.set_xlim(xlim)axes.set_ylim(ylim)if legend:axes.legend(legend)axes.grid()
#绘制学习的情况
class Animator: """在动画中绘制数据"""def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,ylim=None, xscale='linear', yscale='linear',fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,figsize=(3.5, 2.5)):# 增量地绘制多条线if legend is None:legend = []self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)if nrows * ncols == 1:self.axes = [self.axes, ]# 使⽤lambda函数捕获参数self.config_axes = lambda: set_axes(self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)self.X, self.Y, self.fmts = None, None, fmtsdef add(self, x, y):# 向图表中添加多个数据点if not hasattr(y, "__len__"):y = [y]n = len(y)if not hasattr(x, "__len__"):x = [x] * nif not self.X:self.X = [[] for _ in range(n)]if not self.Y:self.Y = [[] for _ in range(n)]for i, (a, b) in enumerate(zip(x, y)):if a is not None and b is not None:self.X[i].append(a)self.Y[i].append(b)self.axes[0].cla()for x, y, fmt in zip(self.X, self.Y, self.fmts):self.axes[0].plot(x, y, fmt)self.config_axes()#display.display(self.fig)# 通过以下两行代码实现了在PyCharm中显示动图plt.draw()#plt.pause(interval=0.001)#display.clear_output(wait=True)#批大小
batch_size = 256
#训练和测试的迭代器
train_iter, test_iter = load_data_fashion_mnist(batch_size)# for X, y in train_iter:
#     print(X.shape, X.dtype, y.shape, y.dtype)
#     breaknum_inputs = 784
num_outputs = 10
#初始化权重和偏置
W = torch.normal(0, 0.01, size=(num_inputs, num_outputs), requires_grad=True)#784*10的权重矩阵
b = torch.zeros(num_outputs, requires_grad=True)# softmax操作
def softmax(X):X_exp = torch.exp(X)partition = X_exp.sum(1, keepdim=True)return X_exp / partition # 这⾥应⽤了⼴播机制#网络
def net(X):return softmax(torch.matmul(X.reshape((-1, W.shape[0])), W) + b)# 损失函数 交叉熵
def cross_entropy(y_hat, y):return - torch.log(y_hat[range(len(y_hat)), y])#精度计算函数
def accuracy(y_hat, y): """计算预测正确的数量"""if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:y_hat = y_hat.argmax(axis=1)cmp = y_hat.type(y.dtype) == yreturn float(cmp.type(y.dtype).sum())class Accumulator: """在n个变量上累加"""def __init__(self, n):self.data = [0.0] * ndef add(self, *args):self.data = [a + float(b) for a, b in zip(self.data, args)]def reset(self):self.data = [0.0] * len(self.data)def __getitem__(self, idx):return self.data[idx]
#精度计算函数搭配accumulator使用
def evaluate_accuracy(net, data_iter):"""计算在指定数据集上模型的精度"""if isinstance(net, torch.nn.Module):net.eval() # 将模型设置为评估模式metric = Accumulator(2) # 正确预测数、预测总数with torch.no_grad():for X, y in data_iter:metric.add(accuracy(net(X), y), y.numel())return metric[0] / metric[1]
#训练单轮
def train_epoch_ch3(net, train_iter, loss, updater): """训练模型⼀个迭代周期（定义⻅第3章）"""# 将模型设置为训练模式if isinstance(net, torch.nn.Module):net.train()# 训练损失总和、训练准确度总和、样本数metric = Accumulator(3)for X, y in train_iter:# 计算梯度并更新参数y_hat = net(X)l = loss(y_hat, y)if isinstance(updater, torch.optim.Optimizer):# 使⽤PyTorch内置的优化器和损失函数updater.zero_grad() #清除梯度l.mean().backward() #反向传播updater.step()else:# 使⽤定制的优化器和损失函数l.sum().backward()updater(X.shape[0])metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())# 返回训练损失和训练精度return metric[0] / metric[2], metric[1] / metric[2]
#训练
def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): #@save"""训练模型（定义⻅第3章）"""animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],legend=['train loss', 'train acc', 'test acc'])for epoch in range(num_epochs):train_metrics = train_epoch_ch3(net, train_iter, loss, updater)test_acc = evaluate_accuracy(net, test_iter)animator.add(epoch + 1, train_metrics + (test_acc,))train_loss, train_acc = train_metricsassert train_loss < 0.5, train_lossassert train_acc <= 1 and train_acc > 0.7, train_accassert test_acc <= 1 and test_acc > 0.7, test_acclr = 0.1
#优化算法 小批量随机梯度下降（Stochastic Gradient Descent）
def sgd(params,lr,batch_size):with torch.no_grad():for param in params:param-=lr*param.grad/batch_sizeparam.grad.zero_()#更新器使用sgd优化算法
def updater(batch_size):return sgd([W, b], lr, batch_size)num_epochs = 10
train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
plt.show()def predict_ch3(net, test_iter, n=6):"""预测标签（定义⻅第3章）"""for X, y in test_iter:breaktrues = get_fashion_mnist_labels(y)preds = get_fashion_mnist_labels(net(X).argmax(axis=1))titles = [true +'\n' + pred for true, pred in zip(trues, preds)]show_images(X[0:n].reshape((n, 28, 28)), 1, n, titles=titles[0:n])predict_ch3(net, test_iter)

课后题：

softmax函数可能会导致什么问题？
softmax运算可能造成溢出，因为分母要计算多个exp的值求和
cross_entropy 是根据交叉熵损失函数的定义实现的。这个实现可能有什么问题？
y^中若某行最大的值也接近0的话，loss的值也可能造成溢出，
什么解决方案来解决上述两个问题？
nllloss和log_softmax一起使用。softmax之后的损失函数是交叉熵，这很合理。但是softmax有可能上溢，交叉熵也有可能溢出(-log0)，所以可以用Log_softmax和负对数似然估计nll loss 配套使用。log_softmax就是对softmax取对数，nll就是直接取反，比交叉熵还方便。

结果：
在这里插入图片描述

调库实现

import torch
from torch import nn
from torchvision import transforms
import torchvision
from torch.utils import data
import matplotlib.pyplot as plt#加载fashion_mnist数据集
def load_data_fashion_mnist(batch_size, resize=None):"""下载Fashion-MNIST数据集，然后将其加载到内存中"""trans = [transforms.ToTensor()]if resize:trans.insert(0, transforms.Resize(resize))trans = transforms.Compose(trans)mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)#print(len(mnist_train),len(mnist_test))return (data.DataLoader(mnist_train, batch_size, shuffle=True),data.DataLoader(mnist_test, batch_size, shuffle=False)) #windows下不能多进程,linux下可以#批大小
batch_size = 256
#训练和测试的迭代器
train_iter, test_iter = load_data_fashion_mnist(batch_size)net=nn.Sequential(nn.Flatten(),nn.Linear(784,10))
#初始化模型参数
def init_weights(m):if type(m)==nn.Linear:nn.init.normal_(m.weight,std=0.01)
net.apply(init_weights)
loss=nn.CrossEntropyLoss(reduction='none')
trainer=torch.optim.SGD(net.parameters(),lr=0.1)def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):"""设置matplotlib的轴"""axes.set_xlabel(xlabel)axes.set_ylabel(ylabel)axes.set_xscale(xscale)axes.set_yscale(yscale)axes.set_xlim(xlim)axes.set_ylim(ylim)if legend:axes.legend(legend)axes.grid()
class Animator: """在动画中绘制数据"""def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,ylim=None, xscale='linear', yscale='linear',fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,figsize=(3.5, 2.5)):# 增量地绘制多条线if legend is None:legend = []self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)if nrows * ncols == 1:self.axes = [self.axes, ]# 使⽤lambda函数捕获参数self.config_axes = lambda: set_axes(self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)self.X, self.Y, self.fmts = None, None, fmtsdef add(self, x, y):# 向图表中添加多个数据点if not hasattr(y, "__len__"):y = [y]n = len(y)if not hasattr(x, "__len__"):x = [x] * nif not self.X:self.X = [[] for _ in range(n)]if not self.Y:self.Y = [[] for _ in range(n)]for i, (a, b) in enumerate(zip(x, y)):if a is not None and b is not None:self.X[i].append(a)self.Y[i].append(b)self.axes[0].cla()for x, y, fmt in zip(self.X, self.Y, self.fmts):self.axes[0].plot(x, y, fmt)self.config_axes()#display.display(self.fig)# 通过以下两行代码实现了在PyCharm中显示动图plt.draw()#plt.pause(interval=0.001)#display.clear_output(wait=True)
#精度计算函数
def accuracy(y_hat, y): """计算预测正确的数量"""if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:y_hat = y_hat.argmax(axis=1)cmp = y_hat.type(y.dtype) == yreturn float(cmp.type(y.dtype).sum())
class Accumulator: """在n个变量上累加"""def __init__(self, n):self.data = [0.0] * ndef add(self, *args):self.data = [a + float(b) for a, b in zip(self.data, args)]def reset(self):self.data = [0.0] * len(self.data)def __getitem__(self, idx):return self.data[idx]
def evaluate_accuracy(net, data_iter):"""计算在指定数据集上模型的精度"""if isinstance(net, torch.nn.Module):net.eval() # 将模型设置为评估模式metric = Accumulator(2) # 正确预测数、预测总数with torch.no_grad():for X, y in data_iter:metric.add(accuracy(net(X), y), y.numel())return metric[0] / metric[1]
#训练单轮
def train_epoch_ch3(net, train_iter, loss, updater): """训练模型⼀个迭代周期（定义⻅第3章）"""# 将模型设置为训练模式if isinstance(net, torch.nn.Module):net.train()# 训练损失总和、训练准确度总和、样本数metric = Accumulator(3)for X, y in train_iter:# 计算梯度并更新参数y_hat = net(X)l = loss(y_hat, y)if isinstance(updater, torch.optim.Optimizer):# 使⽤PyTorch内置的优化器和损失函数updater.zero_grad() #清除梯度l.mean().backward() #反向传播updater.step()else:# 使⽤定制的优化器和损失函数l.sum().backward()updater(X.shape[0])metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())# 返回训练损失和训练精度return metric[0] / metric[2], metric[1] / metric[2]
#训练
def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): #@save"""训练模型（定义⻅第3章）"""animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],legend=['train loss', 'train acc', 'test acc'])for epoch in range(num_epochs):train_metrics = train_epoch_ch3(net, train_iter, loss, updater)test_acc = evaluate_accuracy(net, test_iter)animator.add(epoch + 1, train_metrics + (test_acc,))train_loss, train_acc = train_metricsassert train_loss < 0.5, train_lossassert train_acc <= 1 and train_acc > 0.7, train_accassert test_acc <= 1 and test_acc > 0.7, test_accnum_epochs=10
train_ch3(net,train_iter,test_iter,loss,num_epochs,trainer)
plt.show()