invOS = compressionInvertedConfig.getPostingOutputStream(((IndexOnDisk) destIndex).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) destIndex).getPrefix() + ".inverted"+ compressionInvertedConfig.getStructureFileExtension()); destIndex.addIndexStructure( "inverted", compressionInvertedConfig.getStructureClass().getName(), "org.terrier.structures.IndexOnDisk,java.lang.String,org.terrier.structures.DocumentIndex,java.lang.Class", "index,structureName,document,"+ compressionInvertedConfig.getPostingIteratorClass().getName() ); destIndex.addIndexStructureInputStream( "inverted", compressionInvertedConfig.getStructureInputStreamClass().getName(), "org.terrier.structures.IndexOnDisk,java.lang.String,java.util.Iterator,java.lang.Class", "index,structureName,lexicon-entry-inputstream,"+ compressionInvertedConfig.getPostingIteratorClass().getName()); destIndex.setIndexProperty("index.inverted.fields.count", ""+fieldCount); lexOutStream.close();
"lexicon", index.getPath(), index.getPrefix()); compressionConfig.writeIndexProperties(index, "lexicon-entry-inputstream");
directIndexBuilder = compressionDirectConfig.getPostingOutputStream( ((IndexOnDisk) currentIndex).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) currentIndex).getPrefix() + "." + "direct" + compressionDirectConfig.getStructureFileExtension()); } catch (Exception ioe) { logger.error("Cannot make DirectInvertedOutputStream:", ioe); compressionDirectConfig.writeIndexProperties(currentIndex, "document-inputstream"); if (FieldScore.FIELDS_COUNT > 0)
FSOMapFileLexicon.renameMapFileLexicon("tmplexicon", index.getPath(), index.getPrefix(), "lexicon", index.getPath(), index.getPrefix()); compressionConfig.writeIndexProperties(index, "lexicon-entry-inputstream");
compressionConfig.writeIndexProperties(newIndex, "lexicon-entry-inputstream");
directIndexBuilder = compressionDirectConfig.getPostingOutputStream( currentIndex.getPath() + ApplicationSetup.FILE_SEPARATOR + currentIndex.getPrefix() + "." + "direct" + compressionDirectConfig.getStructureFileExtension()); } catch (Exception ioe) { logger.error("Cannot make PostingOutputStream:", ioe); compressionDirectConfig.writeIndexProperties(currentIndex, "document-inputstream");
dfOutput = compressionDirectConfig.getPostingOutputStream(destIndex.getPath() + ApplicationSetup.FILE_SEPARATOR + destIndex.getPrefix() + ".direct" + compressionDirectConfig.getStructureFileExtension()); } catch (Exception e) { metaBuilder.close(); docidOutput.close(); compressionDirectConfig.writeIndexProperties(destIndex, "document-inputstream");
AbstractPostingOutputStream invOut = compressionConfig.getPostingOutputStream( path + ApplicationSetup.FILE_SEPARATOR + prefix + "." + "inverted" + compressionConfig.getStructureFileExtension());
AbstractPostingOutputStream invOut = compressionConfig.getPostingOutputStream( path + ApplicationSetup.FILE_SEPARATOR + prefix + "." + "inverted" + compressionConfig.getStructureFileExtension());
compressionConfig.writeIndexProperties(index, "lexicon-entry-inputstream");
logger.info("Recompressing inverted structure using " + compressionConfig.toString()); AbstractPostingOutputStream icpw = compressionConfig.getPostingOutputStream( index.getPath() + ApplicationSetup.FILE_SEPARATOR + index.getPrefix() + "." + "tmp-inverted" + compressionConfig.getStructureFileExtension()); compressionConfig.writeIndexProperties(index, "lexicon-entry-inputstream"); logger.info("New inverted file size: "+ Files.length(index.getPath() + ApplicationSetup.FILE_SEPARATOR + index.getPrefix() + "." + "tmp-inverted" + compressionConfig.getStructureFileExtension()));
logger.info("Recompressing inverted structure using " + compressionConfig.toString()); AbstractPostingOutputStream icpw = compressionConfig.getPostingOutputStream( index.getPath() + ApplicationSetup.FILE_SEPARATOR + index.getPrefix() + "." + "tmp-inverted" + compressionConfig.getStructureFileExtension()); compressionConfig.writeIndexProperties(index, "lexicon-entry-inputstream"); logger.info("New inverted file size: "+ Files.length(index.getPath() + ApplicationSetup.FILE_SEPARATOR + index.getPrefix() + "." + "tmp-inverted" + compressionConfig.getStructureFileExtension()));
@SuppressWarnings("unchecked") @Test public void testSimple() throws IOException { Index index = Index.createNewIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX); CompressionConfiguration cc = getConfig("inverted", new String[0], 0,0); AbstractPostingOutputStream pos = cc.getPostingOutputStream(((IndexOnDisk)index).getPath() + "/" + ((IndexOnDisk)index).getPrefix() + ".inverted" + cc.getStructureFileExtension()); Pointer p = pos.writePostings(new ArrayOfBasicIterablePosting(new int[]{0, 1}, new int[]{1,2})); pos.close(); cc.writeIndexProperties(index, "lexicon-entry-inputstream"); index.flush(); assertTrue(index.hasIndexStructure("inverted")); PostingIndex<Pointer> inv = (PostingIndex<Pointer>) index.getIndexStructure("inverted"); IterablePosting ip = inv.getPostings(p); assertNotNull(ip); assertEquals(0, ip.next()); assertEquals(1, ip.getFrequency()); assertEquals(1, ip.next()); assertEquals(2, ip.getFrequency()); index.close(); IndexUtil.deleteIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX); } }
/** Update the index's properties for this structure */ public void writeIndexProperties(Index index, String pointerSourceStream) { index.addIndexStructure( this.structureName, this.getStructureClass().getName(), "org.terrier.structures.IndexOnDisk,java.lang.String,java.lang.Class", "index,structureName,"+ this.getPostingIteratorClass().getName() ); index.addIndexStructureInputStream( this.structureName, this.getStructureInputStreamClass().getName(), "org.terrier.structures.IndexOnDisk,java.lang.String,java.util.Iterator,java.lang.Class", "index,structureName,"+pointerSourceStream+","+ this.getPostingIteratorClass().getName() ); index.setIndexProperty("index."+this.structureName+".fields.count", String.valueOf(this.fieldCount)); index.setIndexProperty("index."+this.structureName+".fields.names", ArrayUtils.join(this.fieldNames, ",")); index.setIndexProperty("index."+this.structureName+".blocks", String.valueOf(this.hasBlocks)); index.setIndexProperty("index."+this.structureName+".blocks.max", String.valueOf(this.maxBlocks)); } }
/** * contructor * @param i * @param _structureName */ public InvertedIndexBuilder(IndexOnDisk i, String _structureName, CompressionConfiguration compressionConfig) { this.index = i; this.structureName = _structureName; this.compressionConfig = compressionConfig; try{ file = compressionConfig.getPostingOutputStream( index.getPath() + ApplicationSetup.FILE_SEPARATOR + index.getPrefix() + "." + structureName + compressionConfig.getStructureFileExtension()); } catch (Exception ioe) { logger.error("creating PostingOutputStream for writing the inverted file : ", ioe); } lexiconOutputStream = LexiconOutputStream.class; }