protected void createDocumentPostings(){ if (FieldScore.FIELDS_COUNT > 0) termsInDocument = new BlockFieldDocumentPostingList(FieldScore.FIELDS_COUNT); else termsInDocument = new BlockDocumentPostingList(); blockId = 0; numOfTokensInBlock = 0; }
/** * Hook method that creates the right type of DocumentTree class. */ protected void createDocumentPostings(){ if (FieldScore.FIELDS_COUNT > 0) termsInDocument = new FieldDocumentPostingList(FieldScore.FIELDS_COUNT); else termsInDocument = new DocumentPostingList(); }
/** * Processing the lexicon after finished creating the * inverted index. */ public void finishedInvertedIndexBuild() { optimiseLexicon(); }
/** * creates the final term code to offset file, and the lexicon hash if enabled. */ protected void createLexidFile() { LexiconBuilder.optimise(destIndex, "lexicon"); }
public void processTerm(String term) { /* null means the term has been filtered out (eg stopwords) */ if (term != null) { //add term to thingy tree termsInDocument.insert(term); numOfTokensInDocument++; } }
public void processTerm(String t) { // null means the term has been filtered out (eg stopwords) if (t != null) { //add term to thingy tree ((BlockDocumentPostingList)termsInDocument).insert(t, blockId); numOfTokensInDocument++; if (++numOfTokensInBlock >= BLOCK_SIZE && blockId < MAX_BLOCKS) { numOfTokensInBlock = 0; blockId++; } } }
@Override protected IterablePosting makePostingIterator(String[] _terms, int[] termIds) { return new fieldPostingIterator(_terms, termIds); }
@Override protected void load_indexer_properties() { super.load_indexer_properties(); BLOCK_SIZE = ApplicationSetup.BLOCK_SIZE; MAX_BLOCKS = ApplicationSetup.MAX_BLOCKS; } }
/** * constructor * @param i * @param _structureName */ public LexiconBuilder(IndexOnDisk i, String _structureName, TermCodes tc) { this(i, _structureName, instantiate(LexiconMap.class), "org.terrier.structures.BasicLexiconEntry", "", "", tc); } /**
@Override protected IterablePosting makePostingIterator(String[] _terms, int[] termIds) { return new blockFieldPostings(_terms, termIds); }
/** Add a single term to the lexicon being built * @param term The String term * @param tf the frequency of the term */ public void addTerm(String term, int tf) { TempLex.insert(term,tf); }
protected IterablePosting makePostingIterator(String[] _terms, int[] termIds) { return new postingIterator(_terms, termIds); }
/** FIXME */ void createDocumentPostings() { if (numFields>0) termsInDocument = new FieldDocumentPostingList(numFields); else termsInDocument = new DocumentPostingList(); }
/** Creates a lexicon index for the specified index * @param index IndexOnDisk to make the lexicon index for * @deprecated use optimise instead */ public static void createLexiconIndex(IndexOnDisk index) throws IOException { optimise(index, "lexicon"); }
public void processTerm(String term) { if (term != null) { docPostings.insert(term); } }
protected void createDocumentPostings(){ if (FieldScore.FIELDS_COUNT > 0) termsInDocument = new BlockFieldDocumentPostingList(FieldScore.FIELDS_COUNT); else termsInDocument = new BlockDocumentPostingList(); blockId = 0; numOfTokensInBlock = 0; }
public void processTerm(String t) { // null means the term has been filtered out (eg stopwords) if (t != null) { //add term to thingy tree ((BlockDocumentPostingList)termsInDocument).insert(t, blockId); numOfTokensInDocument++; if (++numOfTokensInBlock >= BLOCK_SIZE && blockId < MAX_BLOCKS) { numOfTokensInBlock = 0; blockId++; } } }
/** Creates a lexicon hash for the specified index * @param index IndexOnDisk to make the LexiconHash the lexicoin * @deprecated use optimise instead */ public static void createLexiconHash(final IndexOnDisk index) throws IOException { optimise(index, "lexicon"); } /** optimise the lexicon */
public void processTerm(String term) { if (term != null) { termsInDocument.insert(term); numOfTokensInDocument++; } }
/** optimise the lexicon */ public void optimiseLexicon() { optimise(index, defaultStructureName); }