/** * Creates a new IndexWriter that writes to a temporary location. When * {@link #crawlerFinished()} is invoked, this temporary index is moved to * its final destination. */ public void crawlerStarted() { try { tempWriter = new IndexWriter(tempIndexDir, null, true); tempWriter.setUseCompoundFile(compound); } catch (IOException e) { log.error("Error", e); } }
/** * Setting to turn on usage of a compound file. When on, multiple files * for each segment are merged into a single file once the segment creation * is finished. This is done regardless of what directory is in use. * @see IndexWriter#setUseCompoundFile(boolean) * @throws IllegalStateException if the index is closed */ public void setUseCompoundFile(boolean useCompoundFile) { synchronized(directory) { assureOpen(); if (indexWriter != null) { indexWriter.setUseCompoundFile(useCompoundFile); } this.useCompoundFile = useCompoundFile; } }
/** * Setting to turn on usage of a compound file. When on, multiple files * for each segment are merged into a single file once the segment creation * is finished. This is done regardless of what directory is in use. * @see IndexWriter#setUseCompoundFile(boolean) * @throws IllegalStateException if the index is closed */ public void setUseCompoundFile(boolean useCompoundFile) { synchronized(directory) { assureOpen(); if (indexWriter != null) { indexWriter.setUseCompoundFile(useCompoundFile); } this.useCompoundFile = useCompoundFile; } }
private int index(File indexDir, File dataDir, String suffix) throws Exception { IndexWriter index = new IndexWriter(FSDirectory.open(indexDir), new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED); // tweak the settings for your hardware index.setUseCompoundFile(false); index.setRAMBufferSizeMB(256); index.setMergeFactor(30); indexDirectory(index, dataDir, suffix); index.optimize(); int numIndexed = index.maxDoc(); index.close(); // you'll need to update indexDirectory() to keep track of indexed files return numIndexed; }
private int index(File indexDir, File dataDir, String suffix) throws Exception { RAMDirectory ramDir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(ramDir, new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED); indexWriter.setUseCompoundFile(false); indexDirectory(indexWriter, dataDir, suffix); int numIndexed = indexWriter.maxDoc(); indexWriter.optimize(); indexWriter.close(); IndexWriter index = new IndexWriter(FSDirectory.open(indexDir), new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED); index.addIndexesNoOptimize(ramDir); index.optimize(); index.close(); return numIndexed; }
private int index(File indexDir, File dataDir, String suffix) throws Exception { RAMDirectory ramDir = new RAMDirectory(); // 1 IndexWriter indexWriter = new IndexWriter( ramDir, // 2 new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); indexWriter.setUseCompoundFile(false); indexDirectory(indexWriter, dataDir, suffix); int numIndexed = indexWriter.maxDoc(); indexWriter.optimize(); indexWriter.close(); Directory.copy(ramDir, FSDirectory.open(indexDir), false); // 3 return numIndexed; }
private IndexWriter createIndexWriter(Directory dir, IndexDeletionPolicy deletionPolicy, boolean create) throws IOException{ if(deletionPolicy == null) deletionPolicy = new KeepOnlyLastCommitDeletionPolicy(); IndexWriter indexWriter = new IndexWriter(dir, indexAnalyzer, create, deletionPolicy, MaxFieldLength.LIMITED); indexWriter.setMergeScheduler(new SerialMergeScheduler()); indexWriter.setMergeFactor(mergeFactor); indexWriter.setRAMBufferSizeMB(bufferSizeInMB); indexWriter.setUseCompoundFile(useCompoundFile); return indexWriter; }
indexWriter.setUseCompoundFile(m_luceneUseCompoundFile.booleanValue());
/** * Close the IndexReader and open an IndexWriter. * @throws CorruptIndexException if the index is corrupt * @throws LockObtainFailedException if another writer * has this index open (<code>write.lock</code> could not * be obtained) * @throws IOException if there is a low-level IO error */ protected void createIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException { if (indexWriter == null) { if (indexReader != null) { indexReader.close(); indexReader = null; } indexWriter = new IndexWriter(directory, analyzer, false, new IndexWriter.MaxFieldLength(maxFieldLength)); // IndexModifier cannot use ConcurrentMergeScheduler // because it synchronizes on the directory which can // cause deadlock indexWriter.setMergeScheduler(new SerialMergeScheduler()); indexWriter.setInfoStream(infoStream); indexWriter.setUseCompoundFile(useCompoundFile); if (maxBufferedDocs != IndexWriter.DISABLE_AUTO_FLUSH) indexWriter.setMaxBufferedDocs(maxBufferedDocs); indexWriter.setMergeFactor(mergeFactor); } }
/** * Close the IndexReader and open an IndexWriter. * @throws CorruptIndexException if the index is corrupt * @throws LockObtainFailedException if another writer * has this index open (<code>write.lock</code> could not * be obtained) * @throws IOException if there is a low-level IO error */ protected void createIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException { if (indexWriter == null) { if (indexReader != null) { indexReader.close(); indexReader = null; } indexWriter = new IndexWriter(directory, analyzer, false, new IndexWriter.MaxFieldLength(maxFieldLength)); // IndexModifier cannot use ConcurrentMergeScheduler // because it synchronizes on the directory which can // cause deadlock indexWriter.setMergeScheduler(new SerialMergeScheduler()); indexWriter.setInfoStream(infoStream); indexWriter.setUseCompoundFile(useCompoundFile); if (maxBufferedDocs != IndexWriter.DISABLE_AUTO_FLUSH) indexWriter.setMaxBufferedDocs(maxBufferedDocs); indexWriter.setMergeFactor(mergeFactor); } }
writer.setUseCompoundFile(writerUseCompoundFile); writer.setMaxBufferedDocs(writerMaxBufferedDocs); writer.setRAMBufferSizeMB(writerRamBufferSizeMb);
writer.setUseCompoundFile(writerUseCompoundFile); writer.setMaxBufferedDocs(writerMaxBufferedDocs); writer.setRAMBufferSizeMB(writerRamBufferSizeMb);
writer.setUseCompoundFile(mergerUseCompoundFile); writer.setMaxBufferedDocs(mergerMaxBufferedDocs); writer.setRAMBufferSizeMB(mergerRamBufferSizeMb);
writer.setUseCompoundFile(mergerUseCompoundFile); writer.setMaxBufferedDocs(mergerMaxBufferedDocs); writer.setRAMBufferSizeMB(mergerRamBufferSizeMb);
writer.setUseCompoundFile(writerUseCompoundFile); writer.setMaxBufferedDocs(writerMaxBufferedDocs); writer.setRAMBufferSizeMB(writerRamBufferSizeMb);
writer.setUseCompoundFile(writerUseCompoundFile); writer.setMaxBufferedDocs(writerMaxBufferedDocs); writer.setRAMBufferSizeMB(writerRamBufferSizeMb);
writer.setUseCompoundFile(writerUseCompoundFile); writer.setMaxBufferedDocs(writerMaxBufferedDocs); writer.setRAMBufferSizeMB(writerRamBufferSizeMb);
writer.setUseCompoundFile(writerUseCompoundFile); writer.setMaxBufferedDocs(writerMaxBufferedDocs); writer.setRAMBufferSizeMB(writerRamBufferSizeMb);
public IndexWriter openIndexWriter(CompassSettings settings, Directory dir, boolean create, IndexDeletionPolicy deletionPolicy, Analyzer analyzer) throws IOException { if (deletionPolicy == null) { deletionPolicy = searchEngineFactory.getIndexDeletionPolicyManager().createIndexDeletionPolicy(dir); } if (analyzer == null) { analyzer = searchEngineFactory.getAnalyzerManager().getDefaultAnalyzer(); } IndexWriter indexWriter = new IndexWriter(dir, analyzer, create, deletionPolicy, new IndexWriter.MaxFieldLength(luceneSettings.getMaxFieldLength())); indexWriter.setMergePolicy(MergePolicyFactory.createMergePolicy(settings)); indexWriter.setMergeScheduler(MergeSchedulerFactory.create(indexManager, settings)); indexWriter.setMaxMergeDocs(settings.getSettingAsInt(LuceneEnvironment.SearchEngineIndex.MAX_MERGE_DOCS, luceneSettings.getMaxMergeDocs())); indexWriter.setMergeFactor(settings.getSettingAsInt(LuceneEnvironment.SearchEngineIndex.MERGE_FACTOR, luceneSettings.getMergeFactor())); indexWriter.setRAMBufferSizeMB(settings.getSettingAsDouble(LuceneEnvironment.SearchEngineIndex.RAM_BUFFER_SIZE, luceneSettings.getRamBufferSize())); indexWriter.setMaxBufferedDocs(settings.getSettingAsInt(LuceneEnvironment.SearchEngineIndex.MAX_BUFFERED_DOCS, luceneSettings.getMaxBufferedDocs())); indexWriter.setMaxBufferedDeleteTerms(settings.getSettingAsInt(LuceneEnvironment.SearchEngineIndex.MAX_BUFFERED_DELETED_TERMS, luceneSettings.getMaxBufferedDeletedTerms())); indexWriter.setUseCompoundFile(indexManager.getStore().isUseCompoundFile()); indexWriter.setMaxFieldLength(luceneSettings.getMaxFieldLength()); indexWriter.setTermIndexInterval(luceneSettings.getTermIndexInterval()); indexWriter.setSimilarity(searchEngineFactory.getSimilarityManager().getIndexSimilarity()); return indexWriter; } }