/** * Apply the given listener on batches on documents matching the given structured query. * * @param urisReadyListener * @param queryDefinition * @return */ public QueryBatcherJobTicket applyOnStructuredQuery(QueryBatchListener urisReadyListener, StructuredQueryDefinition queryDefinition) { return apply(urisReadyListener, dataMovementManager.newQueryBatcher(queryDefinition)); }
/** * Apply the given listener on batches on documents matching the given raw combined query. * * @param urisReadyListener * @param queryDefinition * @return */ public QueryBatcherJobTicket applyOnRawCombinedQuery(QueryBatchListener urisReadyListener, RawCombinedQueryDefinition queryDefinition) { return apply(urisReadyListener, dataMovementManager.newQueryBatcher(queryDefinition)); }
/** * Apply the given listener on batches on documents matching the given raw structured query. * * @param urisReadyListener * @param queryDefinition * @return */ public QueryBatcherJobTicket applyOnRawStructuredQuery(QueryBatchListener urisReadyListener, RawStructuredQueryDefinition queryDefinition) { return apply(urisReadyListener, dataMovementManager.newQueryBatcher(queryDefinition)); }
/** * Apply the given listener on batches on documents matching the URIs from the given iterator. * * @param urisReadyListener * @param uriIterator * @return */ public QueryBatcherJobTicket applyOnIterator(QueryBatchListener urisReadyListener, Iterator<String> uriIterator) { return apply(urisReadyListener, dataMovementManager.newQueryBatcher(uriIterator)); }
/** * Apply the given listener on batches on documents matching the given string query. * * @param urisReadyListener * @param queryDefinition * @return */ public QueryBatcherJobTicket applyOnStringQuery(QueryBatchListener urisReadyListener, StringQueryDefinition queryDefinition) { return apply(urisReadyListener, dataMovementManager.newQueryBatcher(queryDefinition)); }
public QueryBatcher newQueryBatcher(QueryDefinition query) { if ( query instanceof RawStructuredQueryDefinition ) { return moveMgr.newQueryBatcher((RawStructuredQueryDefinition) query); } else if ( query instanceof StructuredQueryDefinition ) { return moveMgr.newQueryBatcher((StructuredQueryDefinition) query); } else { throw new IllegalStateException("Unsupported query type: " + query.getClass().getName()); } }
@Override public QueryBatcher buildQueryBatcher(DatabaseClient databaseClient, DataMovementManager dataMovementManager) { StructuredQueryDefinition query = databaseClient.newQueryManager().newStructuredQueryBuilder().document(documentUris); return dataMovementManager.newQueryBatcher(query); } }
@Override public QueryBatcher buildQueryBatcher(DatabaseClient databaseClient, DataMovementManager dataMovementManager) { StructuredQueryDefinition query = databaseClient.newQueryManager().newStructuredQueryBuilder().collection(collectionUris); return dataMovementManager.newQueryBatcher(query); } }
@Test public void testFinalize() { // this test can't actually validate that the message is printed, so a human // must check the logging output // Expect something like: // 18:03:52.308 [Finalizer] WARN c.m.c.d.impl.QueryBatcherImpl - QueryBatcher instance "unnamed" was never cleanly stopped. You should call dataMovementManager.stopJob. // moveMgr.newQueryBatcher(new ArrayList<String>().iterator()); }
@Override public QueryBatcher buildQueryBatcher(DatabaseClient databaseClient, DataMovementManager dataMovementManager) { final Iterator<EvalResult> evalResults = databaseClient.newServerEval().xquery(String.format("cts:uri-match('%s')", uriPattern)).eval().iterator(); return dataMovementManager.newQueryBatcher(new EvalResultIterator(evalResults)); } }
@Test public void testPrematureStopIteratorJob() { // this test can't actually validate that the message is printed, so a human // must check the logging output // Expected something like: // 18:14:58.420 [main] WARN c.m.c.d.impl.QueryBatcherImpl - QueryBatcher instance "unnamed" stopped before all results were processed List<String> list = new ArrayList<>(); list.add("firstUri.txt"); QueryBatcher batcher = moveMgr.newQueryBatcher(list.iterator()); moveMgr.stopJob(batcher); }
@Test public void testIterator() throws Exception { Map<String, String[]> matchesByForest = new HashMap<>(); matchesByForest.put("java-unittest-1", new String[] {uri1, uri3, uri4}); matchesByForest.put("java-unittest-2", new String[] {uri5}); matchesByForest.put("java-unittest-3", new String[] {uri2}); String[] uris = new String[] {uri1, uri2, uri3, uri4, uri5}; List<String> uriList = Arrays.asList(uris); runQueryBatcher(moveMgr.newQueryBatcher(uriList.iterator()), null, matchesByForest, 1, 1, false); runQueryBatcher(moveMgr.newQueryBatcher(uriList.iterator()), null, matchesByForest, 2, 2, false); runQueryBatcher(moveMgr.newQueryBatcher(uriList.iterator()), null, matchesByForest, 2, 3, false); runQueryBatcher(moveMgr.newQueryBatcher(uriList.iterator()), null, matchesByForest, 2, 10, false); runQueryBatcher(moveMgr.newQueryBatcher(uriList.iterator()), null, matchesByForest, 10, 1, false); runQueryBatcher(moveMgr.newQueryBatcher(uriList.iterator()), null, matchesByForest, 18, 33, false); }
public void testIteratorExceptions(List<String> uris, int expectedSuccesses, int expectedFailures) { QueryBatcher uriListBatcher = moveMgr.newQueryBatcher(uris.iterator()) .onUrisReady( batch -> { throw new InternalError(errorMessage); } ) .onQueryFailure( queryThrowable -> { throw new InternalError(errorMessage); } ); testExceptions(uriListBatcher, expectedSuccesses, expectedFailures); uriListBatcher = moveMgr.newQueryBatcher(uris.iterator()) .onUrisReady( batch -> { throw new RuntimeException(errorMessage); } ) .onQueryFailure( queryThrowable -> { throw new RuntimeException(errorMessage); } ); testExceptions(uriListBatcher, expectedSuccesses, expectedFailures); }
@Test public void testStructuredQuery() throws Exception { StructuredQueryDefinition query = new StructuredQueryBuilder().collection(qhbTestCollection); Map<String, String[]> matchesByForest = new HashMap<>(); matchesByForest.put("java-unittest-1", new String[] {uri1, uri3, uri4}); matchesByForest.put("java-unittest-2", new String[] {uri5}); matchesByForest.put("java-unittest-3", new String[] {uri2}); runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 1, 2); }
@Test public void testCollectionQuery() throws Exception { StructuredQueryDefinition query = new StructuredQueryBuilder().and(); query.setCollections(qhbTestCollection); Map<String, String[]> matchesByForest = new HashMap<>(); matchesByForest.put("java-unittest-1", new String[] {uri1, uri3, uri4}); matchesByForest.put("java-unittest-2", new String[] {uri5}); matchesByForest.put("java-unittest-3", new String[] {uri2}); runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 2, 1); }
@Test public void testDirectoryQuery() throws Exception { StructuredQueryDefinition query = new StructuredQueryBuilder().and(); query.setDirectory("/QueryBatcherTest"); query.setCollections(qhbTestCollection); Map<String, String[]> matchesByForest = new HashMap<>(); matchesByForest.put("java-unittest-1", new String[] {uri1, uri3, uri4}); matchesByForest.put("java-unittest-2", new String[] {uri5}); matchesByForest.put("java-unittest-3", new String[] {uri2}); runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 3, 2); }
private void testListenerException(QueryBatchListener listener) { final AtomicInteger failureBatchCount = new AtomicInteger(); Iterator<String> iterator = Arrays.asList(new String[] {uri1}).iterator(); QueryBatcher queryBatcher = moveMgr.newQueryBatcher(iterator) .onUrisReady( batch -> logger.debug("uri={}", batch.getItems()[0]) ) .onUrisReady(listener) .onQueryFailure( queryThrowable -> failureBatchCount.incrementAndGet() ); moveMgr.startJob(queryBatcher); queryBatcher.awaitCompletion(); moveMgr.stopJob(queryBatcher); // there should be no failure sent to the QueryBatcher onQueryFailure listeners assertEquals(0, failureBatchCount.get()); }
@Test public void testStringQuery() throws Exception { StringQueryDefinition query = client.newQueryManager().newStringDefinition().withCriteria("John AND dept:HR"); query.setCollections(qhbTestCollection); query.setOptionsName("employees"); Map<String, String[]> matchesByForest = new HashMap<>(); matchesByForest.put("java-unittest-1", new String[] {uri1, uri3, uri4}); matchesByForest.put("java-unittest-2", new String[] {}); matchesByForest.put("java-unittest-3", new String[] {}); runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 99, 17); }
@Test public void testPrematureStopQueryJob() { // this test can't actually validate that the message is printed, so a human // must check the logging output // Expected something like: // 18:20:33.607 [main] WARN c.m.c.d.impl.QueryBatcherImpl - QueryBatcher instance "unnamed" stopped before all results were retrieved StructuredQueryDefinition query = new StructuredQueryBuilder().and(); QueryBatcher batcher = moveMgr.newQueryBatcher(query); moveMgr.stopJob(batcher); }
@Test public void testRawCtsQuery() throws Exception { String ctsQuery = "<cts:directory-query xmlns:cts=\"http://marklogic.com/cts\"><cts:uri>/QueryBatcherTest/</cts:uri></cts:directory-query>"; RawCtsQueryDefinition query = client.newQueryManager().newRawCtsQueryDefinition(new StringHandle().with(ctsQuery).withFormat(Format.XML)).withCriteria("Jane"); Map<String, String[]> matchesByForest = new HashMap<>(); matchesByForest.put("java-unittest-3", new String[] {uri2}); runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 1, 2); ctsQuery = "{ctsquery : {\"directoryQuery\":{\"uris\":[\"/QueryBatcherTest/\"]}}}";; matchesByForest.put("java-unittest-1", new String[] {uri1, uri3, uri4}); matchesByForest.put("java-unittest-2", new String[] {uri5}); query = client.newQueryManager().newRawCtsQueryDefinition(new StringHandle().with(ctsQuery).withFormat(Format.JSON)); runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 1, 2); }