/** Hook method, called when the inverted index is finished - ie the lexicon is finished */ protected void finishedInvertedIndexBuild() { if (invertedIndexBuilder != null) try{ invertedIndexBuilder.close(); } catch (IOException ioe) { logger.warn("Problem closing inverted index builder", ioe); } LexiconBuilder.optimise(currentIndex, "lexicon"); } }
results = scanLexiconForPointers( numberOfPointersPerIteration, lexiconStream, tmpStorage = new TIntArrayList[processTerms][]; codesHashMap = new TIntIntHashMap(processTerms); results = scanLexiconForTerms( processTerms, lexiconStream, displayMemoryUsage(r); traverseDirectFile(codesHashMap, tmpStorage); if (logger.isDebugEnabled()) logger.debug("time to traverse direct file: " + ((System.currentTimeMillis() - startTraversingDirectFile) / 1000D)); displayMemoryUsage(r); _numberOfTokens += writeInvertedFilePart(dos, tmpStorage, processTerms); if (logger.isDebugEnabled()) logger.debug("time to write inverted file: " displayMemoryUsage(r); LexiconOutputStream<String> los = getLexOutputStream("tmplexicon");
invertedIndexBuilder = new InvertedIndexBuilder(currentIndex, "inverted", compressionInvertedConfig); invertedIndexBuilder.createInvertedIndex(); finishedInvertedIndexBuild();
tmpStorageStorage.add(createPointerForTerm(le));
invertedIndexBuilder.createInvertedIndex(); this.finishedInvertedIndexBuild(); try{
logger.info("time to process part of lexicon: " + ((System.currentTimeMillis() - startProcessingLexicon) / 1000D)); InvertedIndexBuilder.displayMemoryUsage(r); logger.info("time to traverse direct file: "+ ((System.currentTimeMillis() - startTraversingDirectFile) / 1000D)); InvertedIndexBuilder.displayMemoryUsage(r); logger.info("time to write inverted file: " + ((System.currentTimeMillis() - startWritingInvertedFile) / 1000D)); InvertedIndexBuilder.displayMemoryUsage(r);
/** Hook method, called when the inverted index is finished - ie the lexicon is finished */ protected void finishedInvertedIndexBuild() { if (invertedIndexBuilder != null) try{ invertedIndexBuilder.close(); } catch (IOException ioe) { logger.warn("Problem closing inverted index builder", ioe); } LexiconBuilder.optimise(currentIndex, "lexicon"); }