Python英文文章词频统计(14份剑桥真题词频统计)

import jieba# 以只读方式打开text（即真题库）text = open('text.txt', 'r', encoding = 'utf-8').read()# len(text)#统一为小写 text = text.lower()# 需要剔除的词汇列表，也可以用记事本的形式，添加一个打开记事本的语句即可# 即 stwlist = [line.strip() for line in open 'stopwords.txt',encoding='utf-8').readlines()]# 这里使用列表stwlist = ['the','a','of','to','end','in','you','is','that','for','on','it','as','your','...','14',      'this','or','20','40','27','30','13','21','26','10','15','22',      '32','31','1','2','4','5','6','7','8','9','0','10','11','12','13',      '12','13','15','16','17','25','33','35','36','18','23','19','24',      '38','29','34','37','000','...............................']# 先进行分词words = jieba.cut(text, cut_all = False, HMM = True)#cut_all:是否采用全模式#HMM：是否采用HMM模型word_ = {}for word in words:  if (word.strip() not in stwlist):    if len(word) > 1:      if word != '/t':        if word != '/r/n': # 计算词频          if word in word_:            word_[word] += 1          else:            word_[word] = 1# 将结果保存为元组word_freq = []for word, freq in word_.items():  word_freq.append((word, freq))# 降序排列word_freq.sort(key = lambda x:x[1], reverse = True)#输出前3500个词汇for i in range(3500):  word, freq = word_freq[i]  print('{0:10}{1:5}'.format(word, freq))