Object o = map.put(s[1], new TermFrequency(Integer.parseInt(s[0]))); if (o != null) { logger.warn(i + " returned " + o + ", " + s[1] + ", " + s[0]);
private void createIdf(Vocabulary voc, int l) { long begin = System.currentTimeMillis(); logger.info("creating idf matrix..."); Iidf = new double[voc.entrySet().size()]; //logger.debug("Iidf.size: " + Iidf.length); // iterates over the types Iterator it = voc.entrySet().iterator(); while (it.hasNext()) { Map.Entry me = (Map.Entry) it.next(); String term = (String) me.getKey(); Vocabulary.TermFrequency tf = (Vocabulary.TermFrequency) me.getValue(); int index = termIndex.get(term); Iidf[index] = log2((double) l / tf.get()); //logger.info(index + ": " + l + "/"+ tf.get() + " = " + Iidf[index]); } // end while //for (int i=0;i<Iidf.length;i++) // logger.info(i + " " + Iidf[i]); long end = System.currentTimeMillis(); logger.info("took " + (end - begin) + " ms"); } // end createIdf
indexes[j] = ti; values[j] = (float) (1 + Math.log(tf.get())); corpusVocabulary.add(term); j++;
private void createIdf(Vocabulary voc, int l) { long begin = System.currentTimeMillis(); logger.info("creating idf vector..."); Iidf = new float[voc.entrySet().size()]; //logger.debug("Iidf.size: " + Iidf.length); // iterates over the types Iterator it = voc.entrySet().iterator(); while (it.hasNext()) { Map.Entry me = (Map.Entry) it.next(); String term = (String) me.getKey(); Vocabulary.TermFrequency tf = (Vocabulary.TermFrequency) me.getValue(); int index = termIndex.get(term); Iidf[index] = (float) log2((double) l / tf.get()); //logger.info(index + ": " + l + "/"+ tf.get() + " = " + Iidf[index]); } // end while //for (int i=0;i<Iidf.length;i++) // logger.info(i + " " + Iidf[i]); long end = System.currentTimeMillis(); logger.info("took " + (end - begin) + " ms"); } // end createIdf
private void createIdf(Vocabulary voc, int l) { long begin = System.currentTimeMillis(); logger.info("creating idf matrix..."); Iidf = new float[voc.entrySet().size()]; //logger.debug("Iidf.size: " + Iidf.length); // iterates over the types Iterator it = voc.entrySet().iterator(); while (it.hasNext()) { Map.Entry me = (Map.Entry) it.next(); String term = (String) me.getKey(); Vocabulary.TermFrequency tf = (Vocabulary.TermFrequency) me.getValue(); int index = termIndex.get(term); Iidf[index] = log2((float) l / tf.get()); //logger.info(index + ": " + l + "/"+ tf.get() + " = " + Iidf[index]); } // end while //for (int i=0;i<Iidf.length;i++) // logger.info(i + " " + Iidf[i]); long end = System.currentTimeMillis(); logger.info("took " + (end - begin) + " ms"); } // end createIdf
/** * Add a token to the index Vocabulary * * @param token the token. */ public void add(String token) { //logger.debug("Vocabulary.add : " + token); TermFrequency tf = map.get(token); if (tf == null) { ////System.out.print("\"" + token + "\"\n"); tf = new TermFrequency(); map.put(token, tf); } else { tf.inc(); } //logger.debug("added " + token + " (" + tf + ")"); } // end add
/** * Add a token to the index Vocabulary * * @param token the token. */ public int get(String token) { //logger.debug("Vocabulary.add : " + f); TermFrequency tf = map.get(token); if (tf == null) { return 0; } return tf.get(); } // end add