/** Creates a new instance of LuceneIndex */ private LuceneIndex (final Language language, final File refCacheRoot) throws IOException { super(language); assert refCacheRoot != null; this.refCacheRoot = refCacheRoot; this.directory = FSDirectory.getDirectory(refCacheRoot, NoLockFactory.getNoLockFactory()); //Locking controlled by rwlock }
/** * Override to use a different {@link Directory} implementation * * You may want to use {@link org.apache.lucene.store.FSDirectory#open} * which is supposed to select an appropriate * local FS implementation based on the current OS. However, we have seen cases * where using this leads to an implementation that hits {@link java.lang.OutOfMemoryError} * when building large indexes. */ protected Directory getDirectoryImplementation(File location) throws IOException { return new SimpleFSDirectory(location, NoLockFactory.getNoLockFactory()); }
protected Directory createLocalDirForIndexWriter(LuceneIndexDefinition definition, String dirName) throws IOException { String indexPath = definition.getIndexPath(); File indexWriterDir = getIndexDir(definition, indexPath, dirName); //By design indexing in Oak is single threaded so Lucene locking //can be disabled Directory dir = FSDirectory.open(indexWriterDir, NoLockFactory.getNoLockFactory()); log.debug("IndexWriter would use {}", indexWriterDir); return dir; }
protected Directory createLocalDirForIndexWriter(LuceneIndexDefinition definition, String dirName) throws IOException { String indexPath = definition.getIndexPath(); File indexWriterDir = getIndexDir(definition, indexPath, dirName); //By design indexing in Oak is single threaded so Lucene locking //can be disabled Directory dir = FSDirectory.open(indexWriterDir, NoLockFactory.getNoLockFactory()); log.debug("IndexWriter would use {}", indexWriterDir); return dir; }
private DocumentSearcher(final ContextualEnvironment environment, final boolean withPrefixing) throws IOException { this.environment = environment; final StoreConfig config = withPrefixing ? StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING : StoreConfig.WITHOUT_DUPLICATES; this.directory = new ExodusDirectory(environment, config, NoLockFactory.getNoLockFactory()); this.analyzer = createAnalyzer(); }
public OakDirectory(NodeBuilder builder, String dataNodeName, LuceneIndexDefinition definition, boolean readOnly, BlobFactory blobFactory, @NotNull ActiveDeletedBlobCollectorFactory.BlobDeletionCallback blobDeletionCallback, boolean streamingWriteEnabled) { this.lockFactory = NoLockFactory.getNoLockFactory(); this.builder = builder; this.dataNodeName = dataNodeName; this.directoryBuilder = readOnly ? builder.getChildNode(dataNodeName) : builder.child(dataNodeName); this.definition = definition; this.readOnly = readOnly; this.fileNames.addAll(getListing()); this.fileNamesAtStart = ImmutableSet.copyOf(this.fileNames); this.indexName = definition.getIndexName(); this.blobFactory = blobFactory; this.blobDeletionCallback = blobDeletionCallback; this.streamingWriteEnabled = streamingWriteEnabled; }
public OakDirectory(NodeBuilder builder, String dataNodeName, LuceneIndexDefinition definition, boolean readOnly, BlobFactory blobFactory, @NotNull ActiveDeletedBlobCollectorFactory.BlobDeletionCallback blobDeletionCallback, boolean streamingWriteEnabled) { this.lockFactory = NoLockFactory.getNoLockFactory(); this.builder = builder; this.dataNodeName = dataNodeName; this.directoryBuilder = readOnly ? builder.getChildNode(dataNodeName) : builder.child(dataNodeName); this.definition = definition; this.readOnly = readOnly; this.fileNames.addAll(getListing()); this.fileNamesAtStart = ImmutableSet.copyOf(this.fileNames); this.indexName = definition.getIndexName(); this.blobFactory = blobFactory; this.blobDeletionCallback = blobDeletionCallback; this.streamingWriteEnabled = streamingWriteEnabled; }
Preconditions.checkArgument(maxMergeFactor > 0); Directory directory = new SimpleFSDirectory(tmpDirFile, NoLockFactory.getNoLockFactory()); IndexWriter writer = LuceneIndexOutputFormat.createIndexWriter( directory,
lockFactory = NoLockFactory.getNoLockFactory(); } else { String lockClassName = System.getProperty("org.apache.lucene.store.FSDirectoryLockFactoryClass");
this.directory = FSDirectory.getDirectory(refCacheRoot, NoLockFactory.getNoLockFactory()); //Locking controlled by rwlock closed = false;
private Directory newIndexDirectory(LuceneIndexDefinition indexDefinition, NodeBuilder definition, String dirName) throws IOException { String path = null; if (FulltextIndexConstants.PERSISTENCE_FILE.equalsIgnoreCase( definition.getString(FulltextIndexConstants.PERSISTENCE_NAME))) { path = definition.getString(FulltextIndexConstants.PERSISTENCE_PATH); } if (path == null) { if (!remoteDirectory()) { return new BufferedOakDirectory(definition, dirName, indexDefinition, blobStore, blobDeletionCallback); } else { return new OakDirectory(definition, dirName, indexDefinition, false, blobStore, blobDeletionCallback); } } else { // try { File file = new File(path); file.mkdirs(); // TODO: no locking used // --> using the FS backend for the index is in any case // troublesome in clustering scenarios and for backup // etc. so instead of fixing these issues we'd better // work on making the in-content index work without // problems (or look at the Solr indexer as alternative) return FSDirectory.open(file, getNoLockFactory()); } } }
@Override public Directory newInstance(LuceneIndexDefinition definition, NodeBuilder builder, String dirName, boolean reindex) throws IOException { File indexDir = DirectoryUtils.createIndexDir(baseDir, definition.getIndexPath()); File readMe = new File(indexDir, INDEX_METADATA_FILE_NAME); File subDir = DirectoryUtils.createSubDir(indexDir, dirName); FileUtils.forceMkdir(subDir); IndexMeta meta; if (!readMe.exists()) { meta = new IndexMeta(definition.getIndexPath()); } else { meta = new IndexMeta(readMe); } meta.addDirectoryMapping(dirName, subDir.getName()); DirectoryUtils.writeMeta(indexDir, meta); return FSDirectory.open(subDir, NoLockFactory.getNoLockFactory()); }
@Override public Directory newInstance(LuceneIndexDefinition definition, NodeBuilder builder, String dirName, boolean reindex) throws IOException { File indexDir = DirectoryUtils.createIndexDir(baseDir, definition.getIndexPath()); File readMe = new File(indexDir, INDEX_METADATA_FILE_NAME); File subDir = DirectoryUtils.createSubDir(indexDir, dirName); FileUtils.forceMkdir(subDir); IndexMeta meta; if (!readMe.exists()) { meta = new IndexMeta(definition.getIndexPath()); } else { meta = new IndexMeta(readMe); } meta.addDirectoryMapping(dirName, subDir.getName()); DirectoryUtils.writeMeta(indexDir, meta); return FSDirectory.open(subDir, NoLockFactory.getNoLockFactory()); }
private Directory newIndexDirectory(LuceneIndexDefinition indexDefinition, NodeBuilder definition, String dirName) throws IOException { String path = null; if (FulltextIndexConstants.PERSISTENCE_FILE.equalsIgnoreCase( definition.getString(FulltextIndexConstants.PERSISTENCE_NAME))) { path = definition.getString(FulltextIndexConstants.PERSISTENCE_PATH); } if (path == null) { if (!remoteDirectory()) { return new BufferedOakDirectory(definition, dirName, indexDefinition, blobStore, blobDeletionCallback); } else { return new OakDirectory(definition, dirName, indexDefinition, false, blobStore, blobDeletionCallback); } } else { // try { File file = new File(path); file.mkdirs(); // TODO: no locking used // --> using the FS backend for the index is in any case // troublesome in clustering scenarios and for backup // etc. so instead of fixing these issues we'd better // work on making the in-content index work without // problems (or look at the Solr indexer as alternative) return FSDirectory.open(file, getNoLockFactory()); } } }
public void configure(CompassSettings settings) throws CompassException { try { String transLogConnection = settings.getSetting(LuceneEnvironment.Transaction.Processor.ReadCommitted.TransLog.CONNECTION, "ram://"); if ("ram://".equals(transLogConnection)) { directory = new RAMDirectory(); } else { if (transLogConnection.equals("file://")) { transLogConnection = DEFAULT_LOCATION; } else if (transLogConnection.startsWith("file://")) { transLogConnection = transLogConnection.substring("file://".length()); } transLogConnection += "/" + transId.nextLong(); directory = FSDirectory.getDirectory(transLogConnection); // TODO we can improve the file system one by starting with a ram one and then switching } // since this is single threaded access, there is no need to have locks directory.setLockFactory(NoLockFactory.getNoLockFactory()); // create an index writer with autoCommit=true since we want it to be visible to readers (still need to flush) indexWriter = searchEngineFactory.getLuceneIndexManager().getIndexWritersManager().openIndexWriter(settings, directory, true, new KeepOnlyLastCommitDeletionPolicy()); // TODO what about merge policy, which one is better? Default is async operations, so good to have merges in the backgournd optimize = settings.getSettingAsBoolean(LuceneEnvironment.Transaction.Processor.ReadCommitted.TransLog.OPTIMIZE_TRANS_LOG, true); } catch (IOException e) { throw new SearchEngineException("Failed to open transactional index for sub index [" + subIndex + "]", e); } }
@Override public Directory createNRTDir(IndexDefinition definition, File indexDir) throws IOException { Directory fsdir = new SimpleFSDirectory(indexDir, NoLockFactory.getNoLockFactory()); //TODO make these configurable return new NRTCachingDirectory(fsdir, 0.001, 0.001); } });
private void reloadIndex() { Directory diskDir; try { synchronized (getLock()) { initializedIndex = false; if(!diskIndex) { resetIndex(); return; } index = new MMapDirectory(directorySnapshot, NoLockFactory.getNoLockFactory()); IndexWriterConfig iwc = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_36, analyzer).setOpenMode(OpenMode.CREATE_OR_APPEND); if (diskIndexRunning) { iwc.setRAMBufferSizeMB(INDEX_RAM_BUFFER_SIZE) .setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES) // TESTING .setMergeScheduler(NoMergeScheduler.INSTANCE); // TETING } writer = new IndexWriter(index, iwc); initializedIndex = (writer.numDocs() > 0) ? true : false; reader = IndexReader.open(getWriter(), true); /* true: apply all deletes (perfhit) */ searcher = new IndexSearcher(reader); needsReset = false; } } catch (IOException e) { System.out.println("Execption reloading index from disk; assuming corrupt. e = " + e); e.printStackTrace(); resetIndex(); } }
index = MMapDirectory.open(directoryRunning, NoLockFactory.getNoLockFactory()); } else { index = new RAMDirectory();
directory.setLockFactory(NoLockFactory.getNoLockFactory());
dir.setLockFactory(NoLockFactory.getNoLockFactory());