private DirectoryLoaderAdaptor getDirectory(final IndexScopedKey indexKey) { final String indexName = indexKey.getIndexName(); return getDirectory(indexName); }
/** * Looks up the Directory adapter if it's already known, or attempts to initialize indexes. */ private DirectoryLoaderAdaptor getDirectory(final String indexName) { DirectoryLoaderAdaptor adapter = openDirectories.get(indexName); if (adapter == null) { synchronized (openDirectories) { adapter = openDirectories.get(indexName); if (adapter == null) { final File path = new File(this.rootDirectory, indexName); final FSDirectory directory = openLuceneDirectory(path); adapter = new DirectoryLoaderAdaptor(directory, indexName, autoChunkSize, affinitySegmentId); openDirectories.put(indexName, adapter); } } } return adapter; }
@Override public Publisher<MarshalledEntry<K, V>> publishEntries(Predicate<? super K> filter, boolean fetchValue, boolean fetchMetadata) { return Flowable.defer(() -> { // Make sure that we update directories before we start iterating upon directories scanForUnknownDirectories(); return Flowable.fromIterable(openDirectories.values()); }) // We parallelize this since the loading below is blocking .parallel() .runOn(Schedulers.from(ctx.getExecutor())) .flatMap(dir -> { final Set<MarshalledEntry<K, V>> allInternalEntries = new HashSet<>(); dir.loadAllEntries(allInternalEntries, Integer.MAX_VALUE, ctx.getMarshaller()); return Flowable.fromIterable(allInternalEntries); }) .filter(me -> filter == null || filter.test(me.getKey())) .sequential(); }
public void testLoadKeyWithInnerNonReadableDir() throws Exception { TestHelper.createIndex(rootDir, indexName, elementCount, true); File innerDir = new File(rootDir.getAbsoluteFile(), "index-B"); try { boolean created = innerDir.mkdir(); assert created; boolean isReadoff = innerDir.setReadable(false); boolean isWriteoff = innerDir.setWritable(false); if (isReadoff && isWriteoff) { LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); cacheLoader.load(5); } else { log.info("Skipping test because it is not possible to make the directory non-readable, i.e. because the tests are run with the root user."); } } catch(Exception ex) { assert ex instanceof PersistenceException; } finally { innerDir.setReadable(true); innerDir.setWritable(true); } }
public void testContainsKeyCacheKeyTypes() throws Exception { TestHelper.createIndex(rootDir, indexName, elementCount, true); LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); assert cacheLoader.contains(new FileListCacheKey(indexName, segmentId)); String[] fileNamesFromIndexDir = TestHelper.getFileNamesFromDir(rootDir, indexName); for(String fileName : fileNamesFromIndexDir) { assert !cacheLoader.contains(new FileReadLockKey(indexName, fileName, segmentId)) : "Failed for " + fileName; assert cacheLoader.contains(new ChunkCacheKey(indexName, fileName, 0, 1024, segmentId)) : "Failed for " + fileName; } assert !cacheLoader.contains(new ChunkCacheKey(indexName, "testFile.txt", 0, 1024, segmentId)); }
@Override public void process(final KeyFilter filter, final CacheLoaderTask task, Executor executor, boolean fetchValue, boolean fetchMetadata) { scanForUnknownDirectories(); ExecutorAllCompletionService eacs = new ExecutorAllCompletionService(executor);
@Test(expectedExceptions = PersistenceException.class) public void testLoadKeyWithNonExistentFile() throws Exception { LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); FileCacheKey key = new FileCacheKey(indexName, "testKey", segmentId); AssertJUnit.assertNull(cacheLoader.load(key)); }
public void testContainsKeyWithNoExistentRootDir() throws IOException { Directory directory = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, indexName).create(); TestHelper.createIndex(rootDir, indexName, elementCount, true); TestHelper.verifyOnDirectory(directory, elementCount, true); String[] fileNamesFromIndexDir = TestHelper.getFileNamesFromDir(rootDir, indexName); LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); for(String fileName : fileNamesFromIndexDir) { FileCacheKey key = new FileCacheKey(indexName, fileName, segmentId); assert cacheLoader.contains(key); //Testing non-existent keys with non-acceptable type assert !cacheLoader.contains(fileName); } }
private DirectoryLoaderAdaptor getDirectory(final IndexScopedKey indexKey) { final String indexName = indexKey.getIndexName(); return getDirectory(indexName); }
/** * Looks up the Directory adapter if it's already known, or attempts to initialize indexes. */ private DirectoryLoaderAdaptor getDirectory(final String indexName) { DirectoryLoaderAdaptor adapter = openDirectories.get(indexName); if (adapter == null) { synchronized (openDirectories) { adapter = openDirectories.get(indexName); if (adapter == null) { final File path = new File(this.rootDirectory, indexName); final FSDirectory directory = openLuceneDirectory(path); adapter = new DirectoryLoaderAdaptor(directory, indexName, autoChunkSize, affinitySegmentId); openDirectories.put(indexName, adapter); } } } return adapter; }
@Override public void process(final KeyFilter filter, final CacheLoaderTask task, Executor executor, boolean fetchValue, boolean fetchMetadata) { scanForUnknownDirectories(); ExecutorAllCompletionService eacs = new ExecutorAllCompletionService(executor);
public void testLoadKey() throws Exception { TestHelper.createIndex(rootDir, indexName, elementCount, true); String[] fileNamesFromIndexDir = TestHelper.getFileNamesFromDir(rootDir, indexName); LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); for(String fileName : fileNamesFromIndexDir) { FileCacheKey key = new FileCacheKey(indexName, fileName, segmentId); AssertJUnit.assertNotNull(cacheLoader.load(key)); //Testing non-existent keys with non-acceptable type AssertJUnit.assertNull(cacheLoader.load(fileName)); } }
@Override public void call() throws Exception { Cache cache = cacheManager.getCache(); Directory directory = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, indexName).create(); TestHelper.createIndex(rootDir, indexName, elementCount, true); TestHelper.verifyOnDirectory(directory, elementCount, true); String[] fileNamesFromIndexDir = TestHelper.getFileNamesFromDir(rootDir, indexName); LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); for(String fileName : fileNamesFromIndexDir) { FileCacheKey key = new FileCacheKey(indexName, fileName, -1); assert cacheLoader.contains(key); //Testing non-existent keys with non-acceptable type assert !cacheLoader.contains(fileName); } } });
private DirectoryLoaderAdaptor getDirectory(final IndexScopedKey indexKey) { final String indexName = indexKey.getIndexName(); return getDirectory(indexName); }
/** * Looks up the Directory adapter if it's already known, or attempts to initialize indexes. */ private DirectoryLoaderAdaptor getDirectory(final String indexName) { DirectoryLoaderAdaptor adapter = openDirectories.get(indexName); if (adapter == null) { synchronized (openDirectories) { adapter = openDirectories.get(indexName); if (adapter == null) { final File path = new File(this.rootDirectory, indexName); final FSDirectory directory = openLuceneDirectory(path); final InternalDirectoryContract wrapped = ContractAdaptorFactory.wrapNativeDirectory(directory); adapter = new DirectoryLoaderAdaptor(wrapped, indexName, autoChunkSize); openDirectories.put(indexName, adapter); } } } return adapter; }
public void testLoadAllKeysWithExclusion() throws Exception { TestHelper.createIndex(rootDir, indexName, elementCount, true); String[] fileNamesFromIndexDir = TestHelper.getFileNamesFromDir(rootDir, indexName); LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); Set keyList = PersistenceUtil.toKeySet(cacheLoader, null); int initialCount = keyList.size(); HashSet exclusionSet = new HashSet(); for(String fileName : fileNamesFromIndexDir) { FileCacheKey key = new FileCacheKey(indexName, fileName, segmentId); AssertJUnit.assertNotNull(cacheLoader.load(key)); exclusionSet.add(key); } keyList = PersistenceUtil.toKeySet(cacheLoader, k -> !exclusionSet.contains(k)); AssertJUnit.assertEquals((initialCount - fileNamesFromIndexDir.length), keyList.size()); Iterator it = keyList.iterator(); if(it.hasNext()) { assert !(it.next() instanceof FileCacheKey); } }
/** * There might be Directories we didn't store yet in the openDirectories Map. * Make sure they are all initialized before serving methods such as {@link #process(org.infinispan.persistence.spi.AdvancedCacheLoader.KeyFilter, org.infinispan.persistence.spi.AdvancedCacheLoader.CacheLoaderTask, java.util.concurrent.Executor, boolean, boolean)} */ private void scanForUnknownDirectories() { File[] filesInRoot = rootDirectory.listFiles(); for (File maybeDirectory : filesInRoot) { if (maybeDirectory.isDirectory()) { String name = maybeDirectory.getName(); try { getDirectory(name); } catch (PersistenceException e) { log.couldNotWalkDirectory(name, e); } } } }
@Test public void testLoadAllKeysWithNullExclusion() throws Exception { TestHelper.createIndex(rootDir, indexName, elementCount, true); String[] fileNamesFromIndexDir = TestHelper.getFileNamesFromDir(rootDir, indexName); LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); Set keyList = PersistenceUtil.toKeySet(cacheLoader, null); for(String fileName : fileNamesFromIndexDir) { FileCacheKey key = new FileCacheKey(indexName, fileName, segmentId); AssertJUnit.assertNotNull(cacheLoader.load(key)); boolean found = false; for(Object keyFromList : keyList) { if(keyFromList instanceof FileCacheKey && keyFromList.equals(key)) { found = true; break; } } assert found : "No corresponding key was found for " + key; } }
/** * There might be Directories we didn't store yet in the openDirectories Map. * Make sure they are all initialized before serving methods such as {@link #publishEntries(Predicate, boolean, boolean)} */ private void scanForUnknownDirectories() { File[] filesInRoot = rootDirectory.listFiles(); if (filesInRoot != null) { for (File maybeDirectory : filesInRoot) { if (maybeDirectory.isDirectory()) { String name = maybeDirectory.getName(); try { getDirectory(name); } catch (PersistenceException e) { log.couldNotWalkDirectory(name, e); } } } } }
public void testLoadAllKeys() throws Exception { TestHelper.createIndex(rootDir, indexName, elementCount, true); String[] fileNamesFromIndexDir = TestHelper.getFileNamesFromDir(rootDir, indexName); LuceneCacheLoader cacheLoader = (LuceneCacheLoader) TestingUtil.getFirstLoader(cacheManager.getCache()); Set keyList = PersistenceUtil.toKeySet(cacheLoader, null); for(String fileName : fileNamesFromIndexDir) { FileCacheKey key = new FileCacheKey(indexName, fileName, segmentId); AssertJUnit.assertNotNull(cacheLoader.load(key)); boolean found = false; for(Object keyFromList : keyList) { if(keyFromList instanceof FileCacheKey && keyFromList.equals(key)) { found = true; break; } } assert found : "No corresponding key was found for " + key; } }