public static Index makeIndex(String[] docnos, String[] documents) throws Exception { count++; return makeIndex(docnos, documents, new BasicIndexer(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + '-'+ count), ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + '-'+ count); }
/** * Constructs an instance of a BasicIndexer, using the given path name * for storing the data structures. * @param path String the path where the data structures will be created. This is assumed to be * absolute. * @param prefix String the filename component of the data structures */ public BasicIndexer(String path, String prefix) { super(path, prefix); //delay the execution of init() if we are a parent class if (this.getClass() == BasicIndexer.class) init(); compressionDirectConfig = CompressionFactory.getCompressionConfiguration("direct", FieldScore.FIELD_NAMES, 0, 0); compressionInvertedConfig = CompressionFactory.getCompressionConfiguration("inverted", FieldScore.FIELD_NAMES, 0, 0); }
metaBuilder = createMetaIndexBuilder(); emptyDocIndexEntry = (FieldScore.FIELDS_COUNT > 0) ? new FieldDocumentIndexEntry(FieldScore.FIELDS_COUNT) : new BasicDocumentIndexEntry(); createDocumentPostings(); indexEmpty(doc.getAllProperties()); indexDocument(doc.getAllProperties(), termsInDocument); logger.info("Rate: "+((double)numberOfDocuments/((double)secs/3600.0d))+" docs/hour"); finishedDirectIndexBuild();
public IndexOnDisk disk(int prefix) { System.err.println("disk()"); ApplicationSetup.setProperty("indexer.meta.forward.keys", "filename"); ApplicationSetup.setProperty("indexer.meta.forward.keylens", "100"); ApplicationSetup.setProperty("indexer.meta.reverse.keys", "filename"); ApplicationSetup.setProperty("termpipelines", ""); Document[] docs1 = new Document[] { new FileDocument("doc1", new ByteArrayInputStream( "curry church turing knuth".getBytes()), new EnglishTokeniser()), new FileDocument("doc2", new ByteArrayInputStream( "turing knuth knuth turing".getBytes()), new EnglishTokeniser()) }; Collection coll = new CollectionDocumentList(docs1, "filename"); BasicIndexer indexer = new BasicIndexer( ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + "-" + String.valueOf(prefix)); assertNotNull(indexer); indexer.createDirectIndex(new Collection[] { coll }); indexer.createInvertedIndex(); IndexOnDisk index = (IndexOnDisk) Index.createIndex( ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + "-" + String.valueOf(prefix)); assertNotNull(index); System.err.println("done"); return index; }
public static Index makeIndexFields(String[] docnos, String[] documents) throws Exception { count++; return makeIndexFields(docnos, documents, new BasicIndexer(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + '-'+ count), ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + '-'+ count); }
public static Index makeIndex(String[] docnos, String[] documents) throws Exception { return makeIndex(docnos, documents, new BasicIndexer(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX)); }
@Test public void testBasicNoFields() throws Exception { ApplicationSetup.setProperty("FieldTags.process", ""); testIndexer(new BasicIndexer(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX), true, false); }
@Test public void testBasicFields() throws Exception { ApplicationSetup.setProperty("FieldTags.process", "TITLE,ELSE"); testIndexer(new BasicIndexer(ApplicationSetup.TERRIER_INDEX_PATH, "fields"), true, true); }