载入用户词典
jieba.load_userdict('userdict.txt')创建停用词表
def stopwordslist(filepath): stopwords=[line.strip() for line in open(filepath,'r',encoding='utf-8').readlines()] return stopwords对句子进行分词
def seg_sentence(sentence): reg = "[^0-9A-Za-z\u4e00-\u9fa5]" sentence = re.sub(reg, "",str(sentence)) sentence_seged=jieba.cut(sentence.strip()) stopwords=stopwordslist('stopwords.txt') #加载停用词表，先去除左右空格 outstr='' for word in sentence_seged: if word not in stopwords: if word !='\t': outstr+=word outstr+=" " return outstr模型训练
def train_model(): #准备语料 df=pd.read_csv('covid_qa.csv',sep='|',usecols=['question']) all_doc=list(df.question) #预处理，抽取所有关键词，放入all_keys中 all_keys=[] for doc in all_doc: key_word=seg_sentence(doc).split() all_keys.append(key_word) ###############################制作语料库############################ #用dictionary方法获取词袋 dictionary=corpora.Dictionary(all_keys) dictionary.save('covid_qa.dic') #使用doc2bow制作语料 bow_corpus=[dictionary.doc2bow(doc) for doc in all_keys] #使用TF-IDF模型对语料库建模 tfidf=models.TfidfModel(bow_corpus) tfidf.save('covid_qa.tfidf') #计算稀疏句子相似度，建立一个索引 index_model=similarities.SparseMatrixSimilarity(tfidf[bow_corpus],num_features=len(dictionary.token2id)) index_model.save('covid_qa.model')问题与答案匹配
def match(question): ############################处理待对比文档########################### target_doc=question #对目标文档分词 target_words=seg_sentence(target_doc).split() #转为词袋表示 dictionary=corpora.Dictionary.load('covid_qa.dic') target_corpus=dictionary.doc2bow(target_words) #计算目标文档的相似度 index_model=similarities.SparseMatrixSimilarity('covid_qa.model') sim=index_model[tfidf[target_corpus]] #根据相似度排序 results=sorted(enumerate(sim),key=lambda item:-item[1]) result=results[0][0]#相似度最高的文档索引 df=pd.read_csv('covid_qa.csv',sep='|',usecols=['answer']) print('问题：',question) print('答案：',df.iloc[result].answer) return df.iloc[result].answer def QA(): print('*'*20,'\n新冠知识问答系统(V1.0)') print('*'*20) question=input('请输入问题(按字母a结束)：') while question !='a': print(match(question)) question=input('请输入问题(按字母a结束)：') train_model()#执行一次就行 QA()