public IndexRepositoryImpl(Region<?, ?> region, IndexWriter writer, LuceneSerializer serializer, LuceneIndexStats stats, Region<?, ?> userRegion, DistributedLockService lockService, String lockName, LuceneIndex index) throws IOException { this.region = region; this.userRegion = userRegion; this.writer = writer; searcherManager = createSearchManager(); this.serializer = serializer; this.stats = stats; documentCountSupplier = new DocumentCountSupplier(); stats.addDocumentsSupplier(documentCountSupplier); this.lockService = lockService; this.lockName = lockName; this.index = index; }
@Override protected IndexRepository getRepository(Integer bucketId) throws BucketNotFoundException { IndexRepository repo = indexRepositories.get(bucketId); if (repo != null && !repo.isClosed()) { return repo; } repo = computeRepository(bucketId); return repo; }
Object callbackArgument = event.getCallbackArgument(); IndexRepository repository = repositoryManager.getRepository(region, key, callbackArgument); repository.update(key, value); } else { repository.delete(key); repo.commit();
private boolean reindexUserDataRegion(Integer bucketId, PartitionedRegion userRegion, PartitionedRegion fileRegion, BucketRegion dataBucket, IndexRepository repo) throws IOException { Set<IndexRepository> affectedRepos = new HashSet<IndexRepository>(); for (Object key : dataBucket.keySet()) { Object value = getValue(userRegion.getEntry(key)); if (value != null) { repo.update(key, value); } else { repo.delete(key); } affectedRepos.add(repo); } for (IndexRepository affectedRepo : affectedRepos) { affectedRepo.commit(); } // fileRegion ops (get/put) need bucketId as a callbackArg for PartitionResolver fileRegion.put(APACHE_GEODE_INDEX_COMPLETE, APACHE_GEODE_INDEX_COMPLETE, bucketId); return true; }
when(mockContext.getArguments()).thenReturn(searchArgs); when(mockContext.getResultSender()).thenReturn(mockResultSender); when(mockRepoManager.getRepositories(eq(mockContext), eq(false))).thenReturn(repos); doAnswer(invocation -> { IndexResultCollector collector = invocation.getArgument(2); collector.collect(r1_1.getKey(), r1_1.getScore()); collector.collect(r1_2.getKey(), r1_2.getScore()); collector.collect(r1_3.getKey(), r1_3.getScore()); return null; }).when(mockRepository1).query(eq(query), eq(LuceneQueryFactory.DEFAULT_LIMIT), any(IndexResultCollector.class)); collector.collect(r2_1.getKey(), r2_1.getScore()); collector.collect(r2_2.getKey(), r2_2.getScore()); return null; }).when(mockRepository2).query(eq(query), eq(LuceneQueryFactory.DEFAULT_LIMIT), any(IndexResultCollector.class));
@Override public IndexRepository computeIndexRepository(final Integer bucketId, LuceneSerializer serializer, InternalLuceneIndex index, PartitionedRegion userRegion, IndexRepository oldRepository, PartitionedRepositoryManager partitionedRepositoryManager) throws IOException { final IndexRepository indexRepo = super.computeIndexRepository(bucketId, serializer, index, userRegion, oldRepository, partitionedRepositoryManager); if (indexRepo == null) { return null; } if (mockingDetails(indexRepo).isSpy()) { return indexRepo; } final IndexRepository spy = Mockito.spy(indexRepo); Answer invokeBeforeWrite = invocation -> { beforeWrite.accept(invocation.getArgument(0)); return invocation.callRealMethod(); }; doAnswer(invokeBeforeWrite).when(spy).update(any(), any()); doAnswer(invokeBeforeWrite).when(spy).create(any(), any()); doAnswer(invokeBeforeWrite).when(spy).delete(any()); return spy; }
@Test(expected = FunctionException.class) public void testIndexRepoQueryFails() throws Exception { when(mockContext.getDataSet()).thenReturn(mockRegion); when(mockContext.getArguments()).thenReturn(searchArgs); when(mockContext.getResultSender()).thenReturn(mockResultSender); when(mockRepoManager.getRepositories(eq(mockContext), eq(false))).thenReturn(repos); doThrow(IOException.class).when(mockRepository1).query(eq(query), eq(LuceneQueryFactory.DEFAULT_LIMIT), any(IndexResultCollector.class)); LuceneQueryFunction function = new LuceneQueryFunction(); function.execute(mockContext); }
@Test public void shouldHandleBucketNotFoundException() throws BucketNotFoundException { Mockito.when(manager.getRepository(any(), any(), any())) .thenThrow(BucketNotFoundException.class); AsyncEvent event = Mockito.mock(AsyncEvent.class); boolean result = listener.processEvents(Arrays.asList(event)); assertFalse(result); verify(listener, times(1)) .logDebugMessage(startsWith("Bucket not found"), any(BucketNotFoundException.class)); }
@Override public void query(Query query, int limit, IndexResultCollector collector) throws IOException { long start = stats.startRepositoryQuery(); int totalHits = 0; IndexSearcher searcher = searcherManager.acquire(); try { TopDocs docs = searcher.search(query, limit); totalHits = docs.totalHits; for (ScoreDoc scoreDoc : docs.scoreDocs) { Document doc = searcher.doc(scoreDoc.doc); Object key = SerializerUtil.getKey(doc); if (logger.isDebugEnabled()) { logger.debug("query found doc:" + doc + ":" + scoreDoc); } collector.collect(key, scoreDoc.score); } } finally { searcherManager.release(searcher); stats.endRepositoryQuery(start, totalHits); } }
@Override public IndexRepository computeIndexRepository(final Integer bucketId, LuceneSerializer serializer, InternalLuceneIndex index, PartitionedRegion userRegion, IndexRepository oldRepository, PartitionedRepositoryManager partitionedRepositoryManager) throws IOException { final IndexRepository repo; if (oldRepository != null) { oldRepository.cleanup(); } LuceneRawIndex indexForRaw = (LuceneRawIndex) index; BucketRegion dataBucket = getMatchingBucket(userRegion, bucketId); Directory dir = null; if (indexForRaw.withPersistence()) { String bucketLocation = LuceneServiceImpl.getUniqueIndexName(index.getName(), index.getRegionPath() + "_" + bucketId); File location = new File(index.getName(), bucketLocation); if (!location.exists()) { location.mkdirs(); } dir = new NIOFSDirectory(location.toPath()); } else { dir = new RAMDirectory(); } IndexWriterConfig config = new IndexWriterConfig(indexForRaw.getAnalyzer()); IndexWriter writer = new IndexWriter(dir, config); return new IndexRepositoryImpl(null, writer, serializer, indexForRaw.getIndexStats(), dataBucket, null, "", indexForRaw); } }
@Override public int getAsInt() { if (isClosed() || !((BucketRegion) userRegion).getBucketAdvisor().isPrimary()) { stats.removeDocumentsSupplier(this); return 0; } try { return writer.numDocs(); } catch (AlreadyClosedException e) { // ignore return 0; } } }
private IndexRepositoryImpl createIndexRepo() throws IOException { ConcurrentHashMap fileAndChunkRegion = new ConcurrentHashMap(); RegionDirectory dir = new RegionDirectory(fileAndChunkRegion, fileSystemStats); IndexWriterConfig config = new IndexWriterConfig(analyzer); IndexWriter writer = new IndexWriter(dir, config); LuceneIndex index = Mockito.mock(LuceneIndex.class); Mockito.when(index.getFieldNames()).thenReturn(new String[] {"txt"}); return new IndexRepositoryImpl(region, writer, mapper, indexStats, null, null, "", index); }
@Test(expected = InternalFunctionInvocationTargetException.class) public void whenCacheIsClosedDuringLuceneQueryExecutionInternalFunctionShouldBeThrownToTriggerFunctionServiceRetry() throws Exception { when(mockContext.getDataSet()).thenReturn(mockRegion); when(mockContext.getArguments()).thenReturn(searchArgs); LuceneQueryFunction function = new LuceneQueryFunction(); when(mockRepoManager.getRepositories(eq(mockContext), eq(false))) .thenThrow(new CacheClosedException()); function.execute(mockContext); }
protected void checkRepository(IndexRepositoryImpl repo0, int... bucketIds) { IndexWriter writer0 = repo0.getWriter(); RegionDirectory dir0 = (RegionDirectory) writer0.getDirectory(); boolean result = false; for (int bucketId : bucketIds) { BucketTargetingMap bucketTargetingMap = new BucketTargetingMap(fileAndChunkBuckets.get(bucketId), bucketId); result |= bucketTargetingMap.equals(dir0.getFileSystem().getFileAndChunkRegion()); } assertTrue(result); assertEquals(serializer, repo0.getSerializer()); }
@Override protected void checkRepository(IndexRepositoryImpl repo0, int... bucketId) { IndexWriter writer0 = repo0.getWriter(); Directory dir0 = writer0.getDirectory(); assertTrue(dir0 instanceof NIOFSDirectory); }
@Test public void injectCustomCollectorManager() throws Exception { final CollectorManager mockManager = mock(CollectorManager.class); searchArgs = new LuceneFunctionContext<IndexResultCollector>(queryProvider, "indexName", mockManager); when(mockContext.getDataSet()).thenReturn(mockRegion); when(mockContext.getArguments()).thenReturn(searchArgs); when(mockContext.getResultSender()).thenReturn(mockResultSender); repos.remove(0); when(mockRepoManager.getRepositories(eq(mockContext), eq(false))).thenReturn(repos); when(mockManager.newCollector(eq("repo2"))).thenReturn(mockCollector); when(mockManager.reduce(any(Collection.class))).thenAnswer(invocation -> { Collection<IndexResultCollector> collectors = invocation.getArgument(0); assertEquals(1, collectors.size()); assertEquals(mockCollector, collectors.iterator().next()); return new TopEntriesCollector(null); }); doAnswer(invocation -> { IndexResultCollector collector = invocation.getArgument(2); collector.collect(r2_1.getKey(), r2_1.getScore()); return null; }).when(mockRepository2).query(eq(query), eq(LuceneQueryFactory.DEFAULT_LIMIT), any(IndexResultCollector.class)); LuceneQueryFunction function = new LuceneQueryFunction(); function.execute(mockContext); verify(mockCollector).collect(eq("key-2-1"), eq(.45f)); verify(mockResultSender).lastResult(any(TopEntriesCollector.class)); }
@Test public void shouldHandleAlreadyClosedException() throws BucketNotFoundException { Mockito.when(manager.getRepository(any(), any(), any())) .thenThrow(AlreadyClosedException.class); AsyncEvent event = Mockito.mock(AsyncEvent.class); boolean result = listener.processEvents(Arrays.asList(event)); assertFalse(result); verify(listener, times(1)) .logDebugMessage(contains("the lucene index is already closed"), any(AlreadyClosedException.class)); }
/** * Return the repository for a given user bucket */ protected IndexRepository getRepository(Integer bucketId) throws BucketNotFoundException { IndexRepository repo = indexRepositories.get(bucketId); if (repo != null && !repo.isClosed()) { return repo; } repo = computeRepository(bucketId); if (repo == null) { throw new BucketNotFoundException( "Unable to find lucene index because no longer primary for bucket " + bucketId); } return repo; }
@Test public void shouldHandleCacheClosedException() throws BucketNotFoundException { Mockito.when(manager.getRepository(any(), any(), any())) .thenThrow(CacheClosedException.class); AsyncEvent event = Mockito.mock(AsyncEvent.class); boolean result = listener.processEvents(Arrays.asList(event)); assertFalse(result); verify(listener, times(1)) .logDebugMessage(contains("cache has been closed"), any(CacheClosedException.class)); }