@SuppressWarnings( "ResultOfMethodCallIgnored" ) @Override public Directory open( File dir ) throws IOException { dir.mkdirs(); FSDirectory directory = USE_DEFAULT_DIRECTORY_FACTORY ? FSDirectory.open( dir.toPath() ) : new NIOFSDirectory( dir.toPath() ); return new NRTCachingDirectory( directory, MAX_MERGE_SIZE_MB, MAX_CACHED_MB ); }
if (doCacheWrite(prefix, context)) { first = cache; second = in; String name = out.getName(); toDelete.add(name); if (slowFileExists(second, name)) { out.close(); } else {
@Override public void sync(Collection<String> fileNames) throws IOException { if (VERBOSE) { System.out.println("nrtdir.sync files=" + fileNames); } for(String fileName : fileNames) { unCache(fileName); } in.sync(fileNames); }
@Override public IndexOutput createOutput(String name, IOContext context) throws IOException { if (VERBOSE) { System.out.println("nrtdir.createOutput name=" + name); } if (doCacheWrite(name, context)) { if (VERBOSE) { System.out.println(" to cache"); } return cache.createOutput(name, context); } else { return in.createOutput(name, context); } }
if (directory instanceof NRTCachingDirectory) { final NRTCachingDirectory nrtCachingDirectory = (NRTCachingDirectory) directory; this.directoryClass = nrtCachingDirectory.getDelegate().getClass().getName(); this.directoryCachedFiles = nrtCachingDirectory.listCachedFiles(); this.directoryCachedRamUsed = FileUtils.byteCountToDisplaySize(nrtCachingDirectory.ramBytesUsed()); } else { this.directoryClass = directory.getClass().getName();
private void unCache(String fileName) throws IOException { // Only let one thread uncache at a time; this only // happens during commit() or close(): synchronized(uncacheLock) { if (VERBOSE) { System.out.println("nrtdir.unCache name=" + fileName); } if (!cache.fileNameExists(fileName)) { // Another thread beat us... return; } assert slowFileExists(in, fileName) == false: "fileName=" + fileName + " exists both in cache and in delegate"; final IOContext context = IOContext.DEFAULT; final IndexOutput out = in.createOutput(fileName, context); IndexInput in = null; try { in = cache.openInput(fileName, context); out.copyBytes(in, in.length()); } finally { IOUtils.close(in, out); } // Lock order: uncacheLock -> this synchronized(this) { // Must sync here because other sync methods have // if (cache.fileNameExists(name)) { ... } else { ... }: cache.deleteFile(fileName); } } }
private Directory reduce(Directory dir) { Directory cdir = dir; if (dir instanceof NRTCachingDirectory) { cdir = ((NRTCachingDirectory)dir).getDelegate(); } if (cdir instanceof TrackingDirectoryWrapper) { cdir = ((TrackingDirectoryWrapper)dir).getDelegate(); } return cdir; } }
@Override public IndexOutput createOutput(String name, IOContext context) throws IOException { if (VERBOSE) { System.out.println("nrtdir.createOutput name=" + name); } if (doCacheWrite(name, context)) { if (VERBOSE) { System.out.println(" to cache"); } return cache.createOutput(name, context); } else { return in.createOutput(name, context); } }
private void unCache(String fileName) throws IOException { // Only let one thread uncache at a time; this only // happens during commit() or close(): synchronized(uncacheLock) { if (VERBOSE) { System.out.println("nrtdir.unCache name=" + fileName); } if (!cache.fileNameExists(fileName)) { // Another thread beat us... return; } assert slowFileExists(in, fileName) == false: "fileName=" + fileName + " exists both in cache and in delegate"; final IOContext context = IOContext.DEFAULT; final IndexOutput out = in.createOutput(fileName, context); IndexInput in = null; try { in = cache.openInput(fileName, context); out.copyBytes(in, in.length()); } finally { IOUtils.close(in, out); } // Lock order: uncacheLock -> this synchronized(this) { // Must sync here because other sync methods have // if (cache.fileNameExists(name)) { ... } else { ... }: cache.deleteFile(fileName); } } }
private Directory reduce(Directory dir) { Directory cdir = dir; if (dir instanceof NRTCachingDirectory) { cdir = ((NRTCachingDirectory)dir).getDelegate(); } if (cdir instanceof TrackingDirectoryWrapper) { cdir = ((TrackingDirectoryWrapper)dir).getDelegate(); } return cdir; }
@Override public Directory createNRTDir(IndexDefinition definition, File indexDir) throws IOException { Directory fsdir = FSDirectory.open(indexDir); //TODO make these configurable return new NRTCachingDirectory(fsdir, 1, 1); } }
@Override public void rename(String source, String dest) throws IOException { unCache(source); if (cache.fileNameExists(dest)) { throw new IllegalArgumentException("target file " + dest + " already exists"); } in.rename(source, dest); }
if (doCacheWrite(prefix, context)) { first = cache; second = in; String name = out.getName(); toDelete.add(name); if (slowFileExists(second, name)) { out.close(); } else {
@Override public IndexOutput createOutput(String name, IOContext context) throws IOException { if (VERBOSE) { System.out.println("nrtdir.createOutput name=" + name); } if (doCacheWrite(name, context)) { if (VERBOSE) { System.out.println(" to cache"); } try { in.deleteFile(name); } catch (IOException ioe) { // This is fine: file may not exist } return cache.createOutput(name, context); } else { try { cache.deleteFile(name); } catch (IOException ioe) { // This is fine: file may not exist } return in.createOutput(name, context); } }
@Override public Directory createNRTDir(IndexDefinition definition, File indexDir) throws IOException { Directory fsdir = FSDirectory.open(indexDir); //TODO make these configurable return new NRTCachingDirectory(fsdir, 1, 1); } }
/** Close this directory, which flushes any cached files * to the delegate and then closes the delegate. */ @Override public void close() throws IOException { // NOTE: technically we shouldn't have to do this, ie, // IndexWriter should have sync'd all files, but we do // it for defensive reasons... or in case the app is // doing something custom (creating outputs directly w/o // using IndexWriter): boolean success = false; try { if (cache.isOpen) { for(String fileName : cache.listAll()) { unCache(fileName); } } success = true; } finally { if (success) { IOUtils.close(cache, in); } else { IOUtils.closeWhileHandlingException(cache, in); } } }
@Override public IndexOutput createOutput(String name, IOContext context) throws IOException { if (VERBOSE) { System.out.println("nrtdir.createOutput name=" + name); } if (doCacheWrite(name, context)) { if (VERBOSE) { System.out.println(" to cache"); } try { in.deleteFile(name); } catch (IOException ioe) { // This is fine: file may not exist } return cache.createOutput(name, context); } else { try { cache.deleteFile(name); } catch (IOException ioe) { // This is fine: file may not exist } return in.createOutput(name, context); } }
static Directory getDirectory(IndexSettingsDefinition settings, Path dataDirectory) throws IOException { final Directory directory = settings == null || settings.directoryType == null || settings.directoryType == IndexSettingsDefinition.Type.FSDirectory ? FSDirectory.open(dataDirectory) : new RAMDirectory(); final double maxMergeSizeMB = settings == null || settings.nrtCachingDirectoryMaxMergeSizeMB == null ? IndexSettingsDefinition.DEFAULT_NRT_CACHING_DIRECTORY_MERGE_SIZE_MB : settings.nrtCachingDirectoryMaxMergeSizeMB; final double maxCacheMB = settings == null || settings.nrtCachingDirectoryMaxCachedMB == null ? IndexSettingsDefinition.DEFAULT_NRT_CACHING_DIRECTORY_MAX_CACHED_MB : settings.nrtCachingDirectoryMaxCachedMB; if (maxMergeSizeMB == 0 || maxCacheMB == 0) return directory; return new NRTCachingDirectory(directory, maxMergeSizeMB, maxCacheMB); }
@Override public void sync(Collection<String> fileNames) throws IOException { if (VERBOSE) { System.out.println("nrtdir.sync files=" + fileNames); } for(String fileName : fileNames) { unCache(fileName); } in.sync(fileNames); }
@Override public Directory createNRTDir(IndexDefinition definition, File indexDir) throws IOException { Directory fsdir = new SimpleFSDirectory(indexDir, NoLockFactory.getNoLockFactory()); //TODO make these configurable return new NRTCachingDirectory(fsdir, 0.001, 0.001); } });