@Override protected void addDirectStructure(IndexOnDisk index) throws Exception { new Inverted2DirectIndexBuilder(index).createDirectIndex(); } }
int countDocsThisIteration = scanDocumentIndexForTokens(processTokens, diis); final Posting[] postings = getPostings(countDocsThisIteration); numberOfTokensFound += traverseInvertedFile(iiis, firstDocid, countDocsThisIteration, postings); logger.info("Writing postings for iteration "+iteration+" to disk"); for (Posting p : postings) //for each document final PostingInRun pir = getPostingReader(); pir.setDf(p.getDocF()); pir.setTF(p.getTF());
@SuppressWarnings("unchecked") @Test public void testScanForTokens() throws Exception { Index index = IndexTestUtils.makeIndex( new String[]{"doc1", "doc2"}, new String[]{"Quick fast brown fox", "jumped huge black lazy dog"});//no stopwords long[] testConstraint = new long[]{0, 1, 4, 5, 9, 10}; int[] numDocs = new int[]{1,1,1,2,2,2}; for(int i=0;i<testConstraint.length;i++) { Iterator<DocumentIndexEntry> dois = (Iterator<DocumentIndexEntry>) index.getIndexStructureInputStream("document"); assertEquals("Looking for numdocs to match " + testConstraint[i] + " tokens", numDocs[i], Inverted2DirectIndexBuilder.scanDocumentIndexForTokens(testConstraint[i], dois)); IndexUtil.close(dois); } }
Inverted2DirectIndexBuilder.main(builderCommand);
Inverted2DirectIndexBuilder.main(builderCommand);
@Override protected void addDirectStructure(IndexOnDisk index) throws Exception { if (! index.hasIndexStructure("direct")) { new Inverted2DirectIndexBuilder(index).createDirectIndex(); } }
@Override public int run(CommandLine line) throws Exception { Index.setIndexLoadingProfileAsRetrieval(false); IndexOnDisk i = Index.createIndex(); if (i== null) { System.err.println("Sorry, no index could be found in default location"); return 1; } Inverted2DirectIndexBuilder i2d = null; //disabling TR-279 optimisation //LexiconBuilder.reAssignTermIds(i, "lexicon", i.getCollectionStatistics().getNumberOfUniqueTerms()); if (line.hasOption('b')) i2d = new BlockInverted2DirectIndexBuilder(i); i2d = new Inverted2DirectIndexBuilder(i); i2d.createDirectIndex(); i.close(); return 0; }
@Test public void testQuickly() throws Exception { Index index = IndexTestUtils.makeIndexSinglePass( new String[]{"doc1", "doc2"}, new String[]{"Quick fast brown fox", "jumped huge black lazy dog"});//no stopwords assertFalse(index.hasIndexStructure("direct")); assertFalse(index.hasIndexStructure("direct-inputstream")); assertTrue(index instanceof IndexOnDisk); new Inverted2DirectIndexBuilder((IndexOnDisk) index).createDirectIndex(); assertTrue(index.hasIndexStructure("direct")); assertTrue(index.hasIndexStructure("direct-inputstream")); }
@Test public void testQuicklyMultiPass() throws Exception { Index index = IndexTestUtils.makeIndexSinglePass( new String[]{"doc1", "doc2"}, new String[]{"Quick fast brown fox", "jumped huge black lazy dog"});//no stopwords assertFalse(index.hasIndexStructure("direct")); assertFalse(index.hasIndexStructure("direct-inputstream")); assertTrue(index instanceof IndexOnDisk); ApplicationSetup.setProperty("inverted2direct.processtokens", "4"); new Inverted2DirectIndexBuilder((IndexOnDisk) index).createDirectIndex(); assertTrue(index.hasIndexStructure("direct")); assertTrue(index.hasIndexStructure("direct-inputstream")); }