TF-IDF理解及其Java实现代码实例

package tfidf;import java.io.*;import java.util.*;import org.wltea.analyzer.lucene.IKAnalyzer;public class ReadFiles {	/**   * @param args   */	private static ArrayList<String> FileList = new ArrayList<String>();	// the list of file	//get list of file for the directory, including sub-directory of it	public static List<String> readDirs(String filepath) throws FileNotFoundException, IOException	  {		try		    {			File file = new File(filepath);			if(!file.isDirectory())			      {				System.out.println("输入的[]");				System.out.println("filepath:" + file.getAbsolutePath());			} else			      {				String[] flist = file.list();				for (int i = 0; i < flist.length; i++)				        {					File newfile = new File(filepath + "//" + flist[i]);					if(!newfile.isDirectory())					          {						FileList.add(newfile.getAbsolutePath());					} else if(newfile.isDirectory()) //if file is a directory, call ReadDirs					{						readDirs(filepath + "//" + flist[i]);					}				}			}		}		catch(FileNotFoundException e)		    {			System.out.println(e.getMessage());		}		return FileList;	}	//read file	public static String readFile(String file) throws FileNotFoundException, IOException	  {		StringBuffer strSb = new StringBuffer();		//String is constant， StringBuffer can be changed.		InputStreamReader inStrR = new InputStreamReader(new FileInputStream(file), "gbk");		//byte streams to character streams		BufferedReader br = new BufferedReader(inStrR);		String line = br.readLine();		while(line != null){			strSb.append(line).append("/r/n");			line = br.readLine();		}		return strSb.toString();	}	//word segmentation	public static ArrayList<String> cutWords(String file) throws IOException{		ArrayList<String> words = new ArrayList<String>();		String text = ReadFiles.readFile(file);		IKAnalyzer analyzer = new IKAnalyzer();		words = analyzer.split(text);		return words;	}	//term frequency in a file, times for each word	public static HashMap<String, Integer> normalTF(ArrayList<String> cutwords){		HashMap<String, Integer> resTF = new HashMap<String, Integer>();		for (String word : cutwords){			if(resTF.get(word) == null){				resTF.put(word, 1);				System.out.println(word);			} else{				resTF.put(word, resTF.get(word) + 1);				System.out.println(word.toString());			}		}		return resTF;	}	//term frequency in a file, frequency of each word	public static HashMap<String, float> tf(ArrayList<String> cutwords){		HashMap<String, float> resTF = new HashMap<String, float>();		int wordLen = cutwords.size();		HashMap<String, Integer> intTF = ReadFiles.normalTF(cutwords);		Iterator iter = intTF.entrySet().iterator();		//iterator for that get from TF		while(iter.hasNext()){			Map.Entry entry = (Map.Entry)iter.next();			resTF.put(entry.getKey().toString(), float.parsefloat(entry.getValue().toString()) / wordLen);			System.out.println(entry.getKey().toString() + " = "+ float.parsefloat(entry.getValue().toString()) / wordLen);		}		return resTF;	}	//tf times for file	public static HashMap<String, HashMap<String, Integer>> normalTFAllFiles(String dirc) throws IOException{		HashMap<String, HashMap<String, Integer>> allNormalTF = new HashMap<String, HashMap<String,Integer>>();		List<String> filelist = ReadFiles.readDirs(dirc);		for (String file : filelist){			HashMap<String, Integer> dict = new HashMap<String, Integer>();			ArrayList<String> cutwords = ReadFiles.cutWords(file);			//get cut word for one file			dict = ReadFiles.normalTF(cutwords);			allNormalTF.put(file, dict);		}		return allNormalTF;	}	//tf for all file	public static HashMap<String,HashMap<String, float>> tfAllFiles(String dirc) throws IOException{		HashMap<String, HashMap<String, float>> allTF = new HashMap<String, HashMap<String, float>>();		List<String> filelist = ReadFiles.readDirs(dirc);		for (String file : filelist){			HashMap<String, float> dict = new HashMap<String, float>();			ArrayList<String> cutwords = ReadFiles.cutWords(file);			//get cut words for one file			dict = ReadFiles.tf(cutwords);			allTF.put(file, dict);		}		return allTF;	}	public static HashMap<String, float> idf(HashMap<String,HashMap<String, float>> all_tf){		HashMap<String, float> resIdf = new HashMap<String, float>();		HashMap<String, Integer> dict = new HashMap<String, Integer>();		int docNum = FileList.size();		for (int i = 0; i < docNum; i++){			HashMap<String, float> temp = all_tf.get(FileList.get(i));			Iterator iter = temp.entrySet().iterator();			while(iter.hasNext()){				Map.Entry entry = (Map.Entry)iter.next();				String word = entry.getKey().toString();				if(dict.get(word) == null){					dict.put(word, 1);				} else {					dict.put(word, dict.get(word) + 1);				}			}		}		System.out.println("IDF for every word is:");		Iterator iter_dict = dict.entrySet().iterator();		while(iter_dict.hasNext()){			Map.Entry entry = (Map.Entry)iter_dict.next();			float value = (float)Math.log(docNum / float.parsefloat(entry.getValue().toString()));			resIdf.put(entry.getKey().toString(), value);			System.out.println(entry.getKey().toString() + " = " + value);		}		return resIdf;	}	public static void tf_idf(HashMap<String,HashMap<String, float>> all_tf,HashMap<String, float> idfs){		HashMap<String, HashMap<String, float>> resTfIdf = new HashMap<String, HashMap<String, float>>();		int docNum = FileList.size();		for (int i = 0; i < docNum; i++){			String filepath = FileList.get(i);			HashMap<String, float> tfidf = new HashMap<String, float>();			HashMap<String, float> temp = all_tf.get(filepath);			Iterator iter = temp.entrySet().iterator();			while(iter.hasNext()){				Map.Entry entry = (Map.Entry)iter.next();				String word = entry.getKey().toString();				float value = (float)float.parsefloat(entry.getValue().toString()) * idfs.get(word);				tfidf.put(word, value);			}			resTfIdf.put(filepath, tfidf);		}		System.out.println("TF-IDF for Every file is :");		DisTfIdf(resTfIdf);	}	public static void DisTfIdf(HashMap<String, HashMap<String, float>> tfidf){		Iterator iter1 = tfidf.entrySet().iterator();		while(iter1.hasNext()){			Map.Entry entrys = (Map.Entry)iter1.next();			System.out.println("FileName: " + entrys.getKey().toString());			System.out.print("{");			HashMap<String, float> temp = (HashMap<String, float>) entrys.getValue();			Iterator iter2 = temp.entrySet().iterator();			while(iter2.hasNext()){				Map.Entry entry = (Map.Entry)iter2.next();				System.out.print(entry.getKey().toString() + " = " + entry.getValue().toString() + ", ");			}			System.out.println("}");		}	}	public static void main(String[] args) throws IOException {		// TODO Auto-generated method stub		String file = "D:/testfiles";		HashMap<String,HashMap<String, float>> all_tf = tfAllFiles(file);		System.out.println();		HashMap<String, float> idfs = idf(all_tf);		System.out.println();		tf_idf(all_tf, idfs);	}}