英文诗歌数据-绘制词云图+本文分类
本项目包含：
1.文本处理
2.词云图绘制
3.文本分类
往期文章可以关注我的专栏
下巴同学的数据加油小站
或者关注CSDN
会不定期分享数据挖掘、机器学习、风控模型、深度学习、NLP等方向的学习项目
数据和完整代码文末链接可以下载
一、导入数据、检查数据
import torch
from torch.utils.data import random_split
import pandas as pd
import numpy as np
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
导入训练集
train_data = pd.read_csv('/home/mw/input/data6487/Poem_classification - train_data.csv')
train_data.head()
Genre Poem
0 Music NaN
1 Music In the thick brushthey spend the...
2 Music Storms are generous. ...
3 Music —After Ana Mendieta Did you carry around the ...
4 Music for Aja Sherrard at 20The portent may itself ...
检查训练集
train_data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 841 entries, 0 to 840
Data columns (total 2 columns):# Column Non-Null Count Dtype
--- ------ -------------- ----- 0 Genre 841 non-null object1 Poem 837 non-null object
dtypes: object(2)
memory usage: 13.3+ KB
train_data = train_data.dropna()
train_data = train_data.reset_index(drop=True)
train_data
Genre Poem
0 Music In the thick brushthey spend the...
1 Music Storms are generous. ...
2 Music —After Ana Mendieta Did you carry around the ...
3 Music for Aja Sherrard at 20The portent may itself ...
4 Music for Bob Marley, Bavaria, November 1980 Here i...
... ... ...
832 Environment Why make so much of fragmentary blue In here a...
833 Environment Woman, I wish I didn't know your name. What co...
834 Environment Yonder to the kiosk, beside the creek, Paddle ...
835 Environment You come to fetch me from my work to-night Whe...
836 Environment You see them through water and glass, (both li...
train_data['Genre'].value_counts()
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inlinefig,ax = plt.subplots(figsize=(6,4), dpi=80)
sns.countplot(x=train_data['Genre'], edgecolor="black", alpha=0.7, data=train_data)
sns.despine()
plt.title("Distribution of the number of poetry genres")
plt.tight_layout()
for p in ax.patches:ax.annotate(f'\n{p.get_height()}', (p.get_x(), p.get_height()+5), color='black', size=10)
测试集部分省略，操作相同
二、词云图
from wordcloud import WordCloud,STOPWORDS,ImageColorGenerator
from PIL import Image
background_image = np.array(Image.open("/home/mw/project/p.jpg"))
fig,ax = plt.subplots(figsize=(12,8), dpi=100)
stopwords = STOPWORDS
stopwords.add('S')
mytext = ''
for i in range(len(test_data)):mytext += test_data['Poem'][i]
for j in range(len(train_data)):mytext += train_data['Poem'][j]
wcloud = WordCloud(width=2400, height=1600,background_color="white",stopwords=stopwords,mask = background_image).generate(mytext)plt.imshow(wcloud)
plt.axis('off')
三、文本分类
处理文本
import re
def handle_data(data):X = data['Poem']y = data['Genre']tv_data = []label = []for i in range(len(X)):r = '[’!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\n。！，—]+'temp = X[i].replace('\n', '')temp = re.sub(r, '', temp)tv_data.append(temp)if y[i] == 'Music':label.append(0)elif y[i] == 'Death':label.append(1)elif y[i] == 'Environment':label.append(2)elif y[i] == 'Affection':label.append(3) return tv_data,label
X_train,y_train = handle_data(train_data)
X_test,y_test = handle_data(test_data)
构建字典
word_list = " ".join(X_train+X_test).split()
word_list = list(set(word_list))
word_dict = {w: i for i, w in enumerate(word_list)}
构建数据集
def transform(sentence, max_len=256):"""把句子转换为数字序列:param sentence::param max_len: 句子的最大长度:return:"""if len(sentence) > max_len:# 句子太长时进行截断sentence = sentence[:max_len]else:# 句子长度不够标准长度时，进行填充sentence = sentence + [0] * (max_len - len(sentence))# print(sentence)return sentencex_input = [np.asarray(transform([word_dict[n] for n in sen.split()])) for sen in X_train]
x_input_test = [np.asarray(transform([word_dict[n] for n in sen.split()])) for sen in X_test]
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoadertrain_inputs = torch.LongTensor(x_input)
train_labels = torch.LongTensor([out for out in y_train])
test_inputs = torch.LongTensor(x_input_test)
test_labels = torch.LongTensor([out for out in y_test])
# 加载训练数据集
dataset = TensorDataset(train_inputs, train_labels)
train_size = int(len(dataset) * 0.8)
val_size = len(dataset) - train_size
train_dataset, val_dataset = random_split(dataset=dataset,lengths=[train_size,val_size],generator=torch.Generator().manual_seed(2022)) #分割验证和训练集
test_dataset = TensorDataset(test_inputs, test_labels)
# # 加载测试数据集
train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True,drop_last=False)
val_loader = DataLoader(val_dataset, batch_size=64, shuffle=True,drop_last=False)
test_loader = DataLoader(test_dataset)模型构建、编译、训练
class MyModel(nn.Module):def __init__(self, vocab_size, embedding_dim, num_filter,filter_sizes, output_dim, dropout=0.5, pad_idx=0):super().__init__()self.embedding = nn.Embedding(vocab_size, embedding_dim)self.convs = nn.ModuleList([nn.Conv2d(in_channels=1, out_channels=num_filter,kernel_size=(fs, embedding_dim))for fs in filter_sizes])# in_channels：输入的channel，文字都是1# out_channels：输出的channel维度# fs：每次滑动窗口计算用到几个单词,相当于n-gram中的n# for fs in filter_sizes用好几个卷积模型最后concate起来看效果。self.fc = nn.Linear(len(filter_sizes) * num_filter, output_dim)self.dropout = nn.Dropout(dropout)def forward(self, text):embedded = self.dropout(self.embedding(text)) # [batch size, sent len, emb dim]embedded = embedded.unsqueeze(1) # [batch size, 1, sent len, emb dim]# 升维是为了和nn.Conv2d的输入维度吻合，把channel列升维。conved = [F.relu(conv(embedded)).squeeze(3) for conv in self.convs]# conved = [batch size, num_filter, sent len - filter_sizes+1]# 有几个filter_sizes就有几个convedpooled = [F.max_pool1d(conv, conv.shape[2]).squeeze(2) for conv in conved] # [batch,num_filter]cat = self.dropout(torch.cat(pooled, dim=1))# cat = [batch size, num_filter * len(filter_sizes)]# 把 len(filter_sizes)个卷积模型concate起来传到全连接层。return self.fc(cat)
vocab_size = len(word_dict) # 词典数量
dmodel = 128 # embedding层词向量num_filter = 100 # 卷积核个数
filter_size = [2, 3, 4] # 卷积核的长
output_dim = 4 # 种类device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# device = torch.device("cpu")model = MyModel(vocab_size+1, dmodel, num_filter=num_filter, filter_sizes=filter_size, output_dim=output_dim).to(device)# 训练循环
def train(dataloader, model, loss_fn, optimizer):size = len(dataloader.dataset) # 训练集的大小num_batches = len(dataloader) # 批次数目, (size/batch_size，向上取整)train_loss, train_acc = 0, 0 # 初始化训练损失和正确率for X, y in dataloader: # 获取图片及其标签X, y = X.to(device), y.to(device)# 计算预测误差pred = model(X) # 网络输出loss = loss_fn(pred, y) # 计算网络输出和真实值之间的差距，targets为真实值，计算二者差值即为损失# 反向传播optimizer.zero_grad() # grad属性归零loss.backward() # 反向传播optimizer.step() # 每一步自动更新# 记录acc与losstrain_acc += (pred.argmax(1) == y).type(torch.float).sum().item()train_loss += loss.item()train_acc /= sizetrain_loss /= num_batchesreturn train_acc, train_lossdef test (dataloader, model, loss_fn):size = len(dataloader.dataset) # 测试集的大小num_batches = len(dataloader) # 批次数目, (size/batch_size，向上取整)test_loss, test_acc = 0, 0# 当不进行训练时，停止梯度更新，节省计算内存消耗with torch.no_grad():for texts, target in dataloader:texts, target = texts.to(device), target.to(device)# 计算losstarget_pred = model(texts)loss = loss_fn(target_pred, target)test_loss += loss.item()test_acc += (target_pred.argmax(1) == target).type(torch.float).sum().item()test_acc /= sizetest_loss /= num_batchesreturn test_acc, test_loss
learn_rate = 1e-2 # 初始学习率
lambda1 = lambda epoch: 0.96 ** (epoch // 5)
optimizer = torch.optim.Adam(model.parameters(), lr=learn_rate)
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda1)
import copyloss_fn = nn.CrossEntropyLoss() # 创建损失函数
epochs = 15train_loss = []
train_acc = []
test_loss = []
test_acc = []best_acc = 0 # 设置一个最佳准确率，作为最佳模型的判别指标for epoch in range(epochs):# 更新学习率（使用自定义学习率时使用）# adjust_learning_rate(optimizer, epoch, learn_rate)model.train()epoch_train_acc, epoch_train_loss = train(train_loader, model, loss_fn, optimizer)scheduler.step() # 更新学习率（调用官方动态学习率接口时使用）model.eval()epoch_test_acc, epoch_test_loss = test(val_loader, model, loss_fn)# 保存最佳模型到 best_modelif epoch_test_acc > best_acc:best_acc = epoch_test_accbest_model = copy.deepcopy(model)train_acc.append(epoch_train_acc)train_loss.append(epoch_train_loss)test_acc.append(epoch_test_acc)test_loss.append(epoch_test_loss)# 获取当前的学习率lr = optimizer.state_dict()['param_groups'][0]['lr']template = ('Epoch:{:2d}, Train_acc:{:.1f}%, Train_loss:{:.3f}, Val_acc:{:.1f}%, Val_loss:{:.3f}, Lr:{:.2E}')print(template.format(epoch+1, epoch_train_acc*100, epoch_train_loss, epoch_test_acc*100, epoch_test_loss, lr))# 保存最佳模型到文件中
PATH = './best_model.pth' # 保存的参数文件名
torch.save(model.state_dict(), PATH)print('Done')
Epoch: 1, Train_acc:24.8%, Train_loss:3.782, Val_acc:23.2%, Val_loss:2.095, Lr:1.00E-02
Epoch: 2, Train_acc:27.5%, Train_loss:2.289, Val_acc:26.2%, Val_loss:1.579, Lr:1.00E-02
Epoch: 3, Train_acc:27.5%, Train_loss:2.020, Val_acc:26.8%, Val_loss:1.707, Lr:1.00E-02
Epoch: 4, Train_acc:31.1%, Train_loss:1.735, Val_acc:28.6%, Val_loss:1.429, Lr:1.00E-02
Epoch: 5, Train_acc:32.9%, Train_loss:1.639, Val_acc:22.6%, Val_loss:1.555, Lr:1.00E-02
Epoch: 6, Train_acc:31.8%, Train_loss:1.616, Val_acc:28.0%, Val_loss:1.427, Lr:1.00E-02
Epoch: 7, Train_acc:35.7%, Train_loss:1.494, Val_acc:34.5%, Val_loss:1.386, Lr:1.00E-02
Epoch: 8, Train_acc:38.4%, Train_loss:1.476, Val_acc:29.2%, Val_loss:1.424, Lr:1.00E-02
Epoch: 9, Train_acc:37.4%, Train_loss:1.487, Val_acc:29.8%, Val_loss:1.526, Lr:1.00E-02
Epoch:10, Train_acc:42.0%, Train_loss:1.448, Val_acc:28.0%, Val_loss:1.500, Lr:1.00E-02
Epoch:11, Train_acc:47.5%, Train_loss:1.397, Val_acc:33.3%, Val_loss:1.682, Lr:1.00E-02
Epoch:12, Train_acc:45.7%, Train_loss:1.489, Val_acc:28.6%, Val_loss:1.716, Lr:1.00E-02
Epoch:13, Train_acc:52.8%, Train_loss:1.295, Val_acc:29.8%, Val_loss:1.511, Lr:1.00E-02
Epoch:14, Train_acc:54.3%, Train_loss:1.202, Val_acc:34.5%, Val_loss:1.620, Lr:1.00E-02
Epoch:15, Train_acc:63.2%, Train_loss:0.997, Val_acc:25.6%, Val_loss:1.785, Lr:1.00E-02
Done
可视化训练损失和精度
过拟合严重，主要是由于数据量太小，参数我也没有仔细调整
使用测试集评估
acc能达到0.4333，还需要调优
model.eval()
epoch_test_acc, epoch_test_loss = test(test_loader, best_model, loss_fn)
epoch_test_acc, epoch_test_loss
(0.43333333333333335, 1.4856029596428078)
数据和完整代码
有需求可以点击下方链接，fork后可以获取所有代码
fork后获取完整代码，可在线运行
数据也上传了csdn一份
数据下载链接
