词频统计及单词提取

对一段英文文本做词频统计，提取单词，查词，最终生成一个单词本，生成的单词本可以导入Anki中学习。

问题分析

考虑到单词的变形，分词后先做词形还原，之后再进行词频统计。去除掉较为简单的单词后，调用金山词霸API查询词义，下载发音音频，生成单词本。同时，用PyQt5做了一个简单的GUI。效果如下：
在这里插入图片描述
打包后生成的软件见: https://pan.baidu.com/s/1fuVdKJWqUqM97jUgTyFMrg 提取码: pstg
核心代码如下：

# -*- coding: utf-8 -*-
'''
分词，提词的主程序
先词形还原，再词频统计
去掉简单的单词后，生成单词本
'''
import requests,re,threading,traceback,os
from bs4 import BeautifulSoup
from tqdm import tqdm
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet
from nltk import pos_tag
lemmatizer = WordNetLemmatizer()
Max_lookup_connections = 5
Max_download_audio_connections = 10
class vocabulary(object):def __init__(self,file,learned_words_file = 'learned_words.txt',save_path='',download_audio=False):''':param: file: 待处理的文本:param: learned_words_file: 已学会的简单的单词本:param: save_path: 要保存的路径，默认程序文件下:param: download_autio:是否下载音频，默认为false'''#生成保存路径self.file = fileif save_path:if not os.path.isdir(save_path):os.makedirs(save_path)else:save_path = './'self.learned_words_file = learned_words_file self.save_path=save_pathself.name = os.path.basename(file).split('.')[0]self.save_filename = self.save_path + os.sep + self.name + '_vocabulary.txt'self.download_audio_flag = download_audiodef run(self):try:raw_text,text = self.get_content(self.file)#获取文本，原始文本用以提供例句words = self.lemmatizing(text)#单词变体还原，词形还原self.word_counts = self.counts(words)#计数words = self.remove_words(words)#去除简单的，已熟知的单词look_up_result = self.get_look_up_result(words,raw_text)#查词self.write_words(look_up_result)if self.download_audio_flag:self.download_audio(look_up_result)return look_up_resultexcept:traceback.print_exc()return Nonedef get_content(self,file):'''获取文本内容，输入文件名，返回字符串。'''with open(file,'r',encoding='utf-8') as f:raw_text = f.read()text = raw_text.lower()for ch in '''`~!@#$%^&*()_+-={}|[]\\:"?>”<;'“—‘’.…/,''':text = text.replace(ch,' ')return raw_text,textdef lemmatizing(self,text):'''词形还原，输入字符串，返回单词列表'''words = text.split()print('words:',len(words))lemm_words = []with tqdm(total = len(words),desc='lemmatizing') as fbar:for i in range(len(words)):j = i+1self.get_lemmed_one(words[i],lemm_words)if j%1000==0:fbar.update(1000)#print('lemm_words: ',len(lemm_words))return lemm_wordsdef get_lemmed_one(self,word,lemm_words):try:tag = pos_tag([word])#标注单词在文本中的成分 #需要用nltk.download('averaged_perceptron_tagger')下载资源pos = self.get_pos(tag[0][1])#转为词性if pos:lemm_word = lemmatizer.lemmatize(word,pos)#词形还原，还原词根lemm_words.append(lemm_word)else:lemm_words.append(word)except:print(word)def get_pos(self,tag):#需要用nltk.download('wordnet')下载资源if tag.startswith('J'):return wordnet.ADJif tag.startswith('V'):return wordnet.VERBif tag.startswith('N'):return wordnet.NOUNif tag.startswith('R'):return wordnet.ADVelse:return ''def counts(self,words):'''词频统计，输入单词列表，输出词频,返回字典{单词:词频}'''#print(len(words))counts = {}for word in words:counts[word] = counts.get(word,0) +1items = list(counts.items())items.sort(key=lambda x:x[1],reverse=True)print('set words:',len(counts))#for i in range(20):#    word,count = items[i]#    print('{0:<10}{1:>5}'.format(word,count))return countsdef remove_words(self,words):learned_words=[]try:with open(self.learned_words_file,'r',encoding='utf-8') as f:for line in f:line = line.replace('\n','')learned_words.append(line)except:learned_words=[]finally:words = list(set(words) - (set(learned_words)))print('removed_words:',len(words))return wordsdef look_up_one(self,word):#查询单个单词，返回：key(单词),ps（音标）,pron（音频url）,pos（词性）,acceptation（释义）#调用金山词霸开放平台API#http://dict-co.iciba.com/api/dictionary.php?w=moose&key=4EE27DDF668AD6501DCC2DC75B46851Burl = 'http://dict-co.iciba.com/api/dictionary.php?w={}&key=4EE27DDF668AD6501DCC2DC75B46851B'.format(word)#print(url)try:resp = requests.get(url)resp.encoding = 'utf-8'soup = BeautifulSoup(resp.text,'html.parser')key = soup.key.stringps = '[{}]'.format(soup.ps.string)pron = soup.pron.stringpos_list = soup.select('pos')pos = pos_list[0].stringacceptation_list = soup.select('acceptation')acceptation = pos_list[0].string + ' ' + acceptation_list[0].string.replace('\n','').replace('\r','')for i in range(1,len(pos_list)):acceptation = acceptation + '<div>' + pos_list[i].string + ' '  + acceptation_list[i].string.replace('\n','').replace('\r','') + '</div>'return (key,ps,pron,pos,acceptation)except:#print(url)#traceback.print_exc()return Nonedef get_sen(self,word,text):#获取原文例句：pattern= '\\..*?{}.*?\\.'.format(word) #问题：大单词包含该小单词match =re.search(pattern,text)if match:return match.group(0)[2:]else:return ' 'def get_look_up_result(self,words,text):'''查词，返回字典列表，key(单词),ps（音标）,pron（音频url）,pos（词性）,acceptation（释义）key: 单词，count: 词频，ps: 音标，pron: 音频url，pos:词性，sen:原文例句，'''data = []threads=[]semaphore = threading.Semaphore(Max_lookup_connections)with tqdm(total = len(words),desc='Looking Up') as fbar:for i in range(len(words)):j = i+1word = words[i]#self.look_up(word,text,data,semaphore)semaphore.acquire()t = threading.Thread(target=self.look_up,args=(word,text,data,semaphore))threads.append(t)t.start()if j%100==0:fbar.update(100)for t in threads:t.join()print('vocabulary:',len(data))#print(data[:10])return datadef look_up(self,word,text,data,semaphore):'''查词，返回字典列表，key(单词),ps（音标）,pron（音频url）,pos（词性）,acceptation（释义）key: 单词，count: 词频，ps: 音标，pron: 音频url，pos:词性，sen:原文例句，'''if self.look_up_one(word):datum = {}key,ps,pron,pos,acceptation=self.look_up_one(word)sen = self.get_sen(word,text)count = self.word_counts.get(key,0)datum['key'] = keydatum['count'] = countdatum['ps'] = psdatum['pron'] = prondatum['pos'] = posdatum['acceptation'] = acceptationdatum['sen'] = sendata.append(datum)semaphore.release()def write_words(self,data):with open(self.save_filename,'w',encoding='utf-8') as f:for datum in data:pron = '[sound:{}.mp3]'.format(datum['key']) text = '{}\t{}\t{}\t{}\t{}\n'.format(datum['key'],datum['ps'],pron,datum['acceptation'],datum['sen'])f.write(text)def download_audio_one(self,key,url):'''下载单词发音音频'''resp = requests.get(url)filepath = self.save_path + os.path.sep + 'audio'if not os.path.isdir(filepath):os.mkdir(filepath)filename = filepath + os.sep + key + '.mp3'if not os.path.isfile(filename):with open(filename,'wb') as f:f.write(resp.content)def download_audio_one_thread(self,key,url,semaphore):'''下载单词发音音频'''resp = requests.get(url)filepath = self.save_path + os.path.sep + 'audio'filename = filepath + os.sep + key + '.mp3'if not os.path.isfile(filename):with open(filename,'wb') as f:f.write(resp.content)semaphore.release()def download_audio(self,data):filepath = self.save_path + os.path.sep + 'audio'if not os.path.isdir(filepath):os.mkdir(filepath)threads = []semaphore = threading.Semaphore(Max_download_audio_connections)len_words = len(data)with tqdm(total = len_words,desc='Downloading sudio') as fbar:for i in range(len_words):j = i+1key = data[i]['key']url = data[i]['pron']semaphore.acquire()t = threading.Thread(target=self.download_audio_one_thread,args=(key,url,semaphore))threads.append(t)t.start()if j%100==0:fbar.update(100)for t in threads:t.join()
if __name__=='__main__':from pandas import DataFramefile = 'Bad Blood - John Carreyrou_1.txt'work =  vocabulary(file,save_path = '')data = work.run()#pf = DataFrame(data)#print(pf)

生成GUI代码为：

# -*- coding: utf-8 -*-
from PyQt5.QtWidgets import QApplication,QMainWindow,QFileDialog,QTableWidgetItem
from PyQt5.QtGui import QFont,QColor
from PyQt5.QtCore import Qt,QDir
from 文本提取单词_UI import Ui_MainWindowfrom vocabulary_2 import vocabulary
from pandas import DataFrame
import traceback
import threading
import os
class MainWindow(QMainWindow,Ui_MainWindow):def __init__(self,parent=None):super(MainWindow,self).__init__(parent)self.setupUi(self)self.file = ''self.learned_word_file = ''self.savepath = './'self.setWindowTitle('文本提词')#连接信号与槽self.pushButton_openfile.clicked.connect(self.openfile)#打开文本文件(utf-8)self.pushButton_savepath.clicked.connect(self.change_save_path)#改变保存路径self.pushButton_learnedfile.clicked.connect(self.openlearnedfile)#打开简单词汇文件（utf-8）self.pushButton_start.clicked.connect(self.run)#生成单词本self.tableWidget.cellClicked.connect(self.listen_one_new)#播放发音self.pushButton_download_audio.clicked.connect(self.download_audio)def openfile(self):#打开文件filename, _ = QFileDialog.getOpenFileName(self,'Open file','D:\\','Txt files(*.txt)')#print(filename)self.label_openfile.setText(filename)self.file = filenamedef change_save_path(self):#修改保存路径directory = QFileDialog.getExistingDirectory(self,'选取文件夹',self.savepath)self.savepath = QDir.toNativeSeparators(directory)# 路径以windows支持的显示方式进行显示。self.label_savepath.setText(self.savepath)def openlearnedfile(self):filename, _ = QFileDialog.getOpenFileName(self,'Open file','D:\\','Txt files(*.txt)')self.label_learnedfile.setText(filename)self.learned_word_file = filenamedef run(self):self.statusBar().showMessage('正在查词......')self.vocabulary = vocabulary(file=self.file,learned_words_file=self.learned_word_file,save_path=self.savepath)data = self.vocabulary.run()if data:self.show_tablewidget(data)self.vocabulary.write_words(self.data)def show_tablewidget(self, dict_data):'''在tableWidget显示dict_data'''tableWidget = self.tableWidget'''排序'''df = DataFrame(dict_data).sort_values(by='count',ascending = False)_temp = df.to_dict('index')dict_data = list(_temp.values())self.data = dict_data'''tableWidget的初始化'''list_col = ['key','count','ps','pron','pos','acceptation','sen']len_col = len(list_col)len_index = len(dict_data)tableWidget.setRowCount(len_index)#设置行数tableWidget.setColumnCount(len_col)#设置列数tableWidget.setHorizontalHeaderLabels(['单词', '词频', '音标','发音','词性','释义','例句']) # 设置垂直方向上的名字tableWidget.setVerticalHeaderLabels([str(i) for i in range(1, len_index + 1)]) # 设置水平方向上的名字'''填充数据'''for index in  range(len_index):for col in range(len_col):name_col = list_col[col]if name_col == 'pron':item = QTableWidgetItem('播放')item.setTextAlignment(Qt.AlignCenter)font = QFont()font.setBold(True)font.setWeight(75)item.setFont(font)item.setBackground(QColor(218, 218, 218))item.setFlags(Qt.ItemIsUserCheckable | Qt.ItemIsEnabled)tableWidget.setItem(index, col, item)else:tableWidget.setItem(index,col,QTableWidgetItem(str(dict_data[index][name_col])))tableWidget.resizeColumnsToContents()tableWidget.setColumnWidth(5, 500)def listen_one_new(self,row,column):if column == 3:download_one = self.data[row]listen_thread = threading.Thread(target = self.listen_one_new_thread,args=(download_one,),daemon=True)listen_thread.start()def listen_one_new_thread(self,download_one):key = download_one['key']url = download_one['pron']self.vocabulary.download_audio_one(key,url)filename = self.savepath + os.path.sep + 'audio' + os.sep + key + '.mp3'print(os.path.abspath(filename))os.system(os.path.abspath(filename))def download_audio(self):self.statusBar().showMessage('音频下载中...')try:self.vocabulary.download_audio(self.data)self.statusBar().showMessage('音频下载成功')except:self.statusBar().showMessage('音频下载失败')if __name__=='__main__':import sysapp = QApplication(sys.argv)window = MainWindow()window.show()sys.exit(app.exec_())

UI代码为：

# -*- coding: utf-8 -*-# Form implementation generated from reading ui file '文本提取单词.ui'
#
# Created by: PyQt5 UI code generator 5.11.3
#
# WARNING! All changes made in this file will be lost!from PyQt5 import QtGui, QtWidgets
from PyQt5.QtCore import QRect, QMetaObject,QCoreApplication
from PyQt5.QtWidgets import QWidget,QVBoxLayout,QHBoxLayout,QPushButton,QLabel,QSpacerItem,QSizePolicy,QTableWidget,QStatusBar
class Ui_MainWindow(object):def setupUi(self, MainWindow):MainWindow.setObjectName("MainWindow")MainWindow.resize(833, 594)self.centralwidget = QWidget(MainWindow)self.centralwidget.setObjectName("centralwidget")self.verticalLayout = QVBoxLayout(self.centralwidget)self.verticalLayout.setObjectName("verticalLayout")self.horizontalLayout = QHBoxLayout()self.horizontalLayout.setObjectName("horizontalLayout")self.pushButton_openfile = QPushButton(self.centralwidget)self.pushButton_openfile.setObjectName("pushButton_openfile")self.horizontalLayout.addWidget(self.pushButton_openfile)self.label_openfile = QLabel(self.centralwidget)self.label_openfile.setText("")self.label_openfile.setObjectName("label_openfile")self.horizontalLayout.addWidget(self.label_openfile)spacerItem = QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum)self.horizontalLayout.addItem(spacerItem)self.pushButton_start = QPushButton(self.centralwidget)self.pushButton_start.setObjectName("pushButton_start")self.horizontalLayout.addWidget(self.pushButton_start)self.verticalLayout.addLayout(self.horizontalLayout)self.horizontalLayout_2 = QHBoxLayout()self.horizontalLayout_2.setObjectName("horizontalLayout_2")self.pushButton_savepath = QPushButton(self.centralwidget)self.pushButton_savepath.setObjectName("pushButton_savepath")self.horizontalLayout_2.addWidget(self.pushButton_savepath)self.label_savepath = QLabel(self.centralwidget)self.label_savepath.setText("")self.label_savepath.setObjectName("label_savepath")self.horizontalLayout_2.addWidget(self.label_savepath)spacerItem1 = QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum)self.horizontalLayout_2.addItem(spacerItem1)self.pushButton_learnedfile = QPushButton(self.centralwidget)self.pushButton_learnedfile.setObjectName("pushButton_learnedfile")self.horizontalLayout_2.addWidget(self.pushButton_learnedfile)self.label_learnedfile = QLabel(self.centralwidget)self.label_learnedfile.setText("")self.label_learnedfile.setObjectName("label_learnedfile")self.horizontalLayout_2.addWidget(self.label_learnedfile)spacerItem2 = QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum)self.horizontalLayout_2.addItem(spacerItem2)self.pushButton_download_audio = QPushButton(self.centralwidget)self.pushButton_download_audio.setObjectName("pushButton_download_audio")self.horizontalLayout_2.addWidget(self.pushButton_download_audio)self.verticalLayout.addLayout(self.horizontalLayout_2)self.tableWidget = QTableWidget(self.centralwidget)self.tableWidget.setObjectName("tableWidget")self.tableWidget.setColumnCount(0)self.tableWidget.setRowCount(0)self.verticalLayout.addWidget(self.tableWidget)MainWindow.setCentralWidget(self.centralwidget)self.statusbar = QStatusBar(MainWindow)self.statusbar.setObjectName("statusbar")MainWindow.setStatusBar(self.statusbar)self.retranslateUi(MainWindow)QMetaObject.connectSlotsByName(MainWindow)def retranslateUi(self, MainWindow):_translate = QCoreApplication.translateMainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))self.pushButton_openfile.setText(_translate("MainWindow", "打开文件"))self.pushButton_start.setText(_translate("MainWindow", "开始"))self.pushButton_savepath.setText(_translate("MainWindow", "保存路径"))self.pushButton_learnedfile.setText(_translate("MainWindow", "需要剔除的单词"))self.pushButton_download_audio.setText(_translate("MainWindow", "下载音频"))