@Override public void waitForCompletion() { if (writeBatcher != null) { writeBatcher.flushAndWait(); if (jobTicket != null) { dataMovementManager.stopJob(writeBatcher); } else { dataMovementManager.stopJob(writeBatcher); } } }
private synchronized boolean processException(Batcher batcher, Throwable throwable, String host) { return (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) ? processGatewayException(batcher, throwable, host) : processForestHostException(batcher, throwable, host); }
@Override public void initialize() { writeBatcher = this.dataMovementManager.newWriteBatcher(); writeBatcher.withThreadCount(threadCount); writeBatcher.withBatchSize(batchSize); jobTicket = dataMovementManager.startJob(writeBatcher); }
public void testExceptions(QueryBatcher queryBatcher, int expectedSuccesses, int expectedFailures) { final AtomicInteger successfulBatchCount = new AtomicInteger(); final AtomicInteger failureBatchCount = new AtomicInteger(); queryBatcher .withBatchSize(1) .onUrisReady( batch -> successfulBatchCount.incrementAndGet() ) .onQueryFailure( queryThrowable -> failureBatchCount.incrementAndGet() ); moveMgr.startJob(queryBatcher); queryBatcher.awaitCompletion(); moveMgr.stopJob(queryBatcher); assertEquals(expectedSuccesses, successfulBatchCount.get()); assertEquals(expectedFailures, failureBatchCount.get()); }
@Test public void testIssue793() { WriteBatcher batcher = moveMgr.newWriteBatcher(); batcher.addAs("test.txt", "test"); moveMgr.startJob(batcher); moveMgr.stopJob(batcher); } }
@Test public void testWithInvalidHosts() throws Exception{ if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) return; ForestConfiguration forestConfig = moveMgr.readForestConfig(); String host1 = forestConfig.listForests()[0].getPreferredHost(); FilteredForestConfiguration ffg = new FilteredForestConfiguration(forestConfig) .withRenamedHost("someInvalidHostName", "anotherInvalidHostName"); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withBlackList("someInvalidHostName"); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withWhiteList("someInvalidHostName") .withWhiteList(host1); runWithWriteAndQueryBatcher(ffg); }
private QueryBatcher newQueryBatcher(QueryBatcherImpl batcher) { // add a default listener to handle host failover scenarios batcher.onQueryFailure(new HostAvailabilityListener(this)); QueryJobReportListener queryJobListener = new QueryJobReportListener(); batcher.onQueryFailure(queryJobListener); batcher.onQueryFailure(new NoResponseListener(this)); batcher.onUrisReady(queryJobListener); return batcher; }
public String getSummaryReport() { JobReport report = moveMgr.getJobReport(ticket); return "batches: " + report.getSuccessBatchesCount() + ", docs: " + report.getSuccessEventsCount() + ", failures: " + report.getFailureEventsCount(); } }
/** * This implementation of initializeListener adds this instance of * ExportToWriterListener to the two RetryListener's in this QueryBatcher so they * will retry any batches that fail during the read request. */ @Override public void initializeListener(QueryBatcher queryBatcher) { HostAvailabilityListener hostAvailabilityListener = HostAvailabilityListener.getInstance(queryBatcher); if ( hostAvailabilityListener != null ) { BatchFailureListener<QueryBatch> retryListener = hostAvailabilityListener.initializeRetryListener(this); if ( retryListener != null ) onFailure(retryListener); } NoResponseListener noResponseListener = NoResponseListener.getInstance(queryBatcher); if ( noResponseListener != null ) { BatchFailureListener<QueryBatch> noResponseRetryListener = noResponseListener.initializeRetryListener(this); if ( noResponseRetryListener != null ) onFailure(noResponseRetryListener); } }
/** * This implementation of initializeListener adds this instance of * UrisToWriterListener to the two RetryListener's in this QueryBatcher so they * will retry any batches that fail during the uris request. */ @Override public void initializeListener(QueryBatcher queryBatcher) { HostAvailabilityListener hostAvailabilityListener = HostAvailabilityListener.getInstance(queryBatcher); if ( hostAvailabilityListener != null ) { BatchFailureListener<QueryBatch> retryListener = hostAvailabilityListener.initializeRetryListener(this); if ( retryListener != null ) onFailure(retryListener); } NoResponseListener noResponseListener = NoResponseListener.getInstance(queryBatcher); if ( noResponseListener != null ) { BatchFailureListener<QueryBatch> noResponseRetryListener = noResponseListener.initializeRetryListener(this); if ( noResponseRetryListener != null ) onFailure(noResponseRetryListener); } }
/** * This implementation of initializeListener adds this instance of * DeleteListener to the two RetryListener's in this QueryBatcher so they * will retry any batches that fail during the delete request. */ @Override public void initializeListener(QueryBatcher queryBatcher) { HostAvailabilityListener hostAvailabilityListener = HostAvailabilityListener.getInstance(queryBatcher); if ( hostAvailabilityListener != null ) { BatchFailureListener<QueryBatch> retryListener = hostAvailabilityListener.initializeRetryListener(this); if ( retryListener != null ) onFailure(retryListener); } NoResponseListener noResponseListener = NoResponseListener.getInstance(queryBatcher); if ( noResponseListener != null ) { BatchFailureListener<QueryBatch> noResponseRetryListener = noResponseListener.initializeRetryListener(this); if ( noResponseRetryListener != null ) onFailure(noResponseRetryListener); } }
public void prepareBatcher(Batcher batcher) { if (jobName != null) { batcher.withJobName(jobName); } if (batchSize != null && batchSize > 0) { batcher.withBatchSize(batchSize); } if (threadCount != null && threadCount > 0) { batcher.withThreadCount(threadCount); } if (forestConfig != null) { batcher.withForestConfig(forestConfig); } }
/** * This implementation of initializeListener adds this instance of * ExportListener to the two RetryListener's in this QueryBatcher so they * will retry any batches that fail during the read request. */ @Override public void initializeListener(QueryBatcher queryBatcher) { HostAvailabilityListener hostAvailabilityListener = HostAvailabilityListener.getInstance(queryBatcher); if ( hostAvailabilityListener != null ) { BatchFailureListener<QueryBatch> retryListener = hostAvailabilityListener.initializeRetryListener(this); if ( retryListener != null ) onFailure(retryListener); } NoResponseListener noResponseListener = NoResponseListener.getInstance(queryBatcher); if ( noResponseListener != null ) { BatchFailureListener<QueryBatch> noResponseRetryListener = noResponseListener.initializeRetryListener(this); if ( noResponseRetryListener != null ) onFailure(noResponseRetryListener); } }
/** * A utility method to return the list of hosts a Batcher should use when talking * to this database. The list is retrieved by calling getPreferredHost() on each * Forest. * * @return the list of hosts a Batcher should use */ public default String[] getPreferredHosts() { return Stream.of(listForests()).map( (forest) -> forest.getPreferredHost()).distinct().toArray(String[]::new); } }
@Override public void write(List<? extends DocumentWriteOperation> items) { for (DocumentWriteOperation op : items) { writeBatcher.add(op.getUri(), op.getMetadata(), op.getContent()); } }
/** * This implementation of initializeListener adds this instance of * ApplyTransformListener to the two RetryListener's in this QueryBatcher so * they will retry any batches that fail during the apply-transform request. */ @Override public void initializeListener(QueryBatcher queryBatcher) { HostAvailabilityListener hostAvailabilityListener = HostAvailabilityListener.getInstance(queryBatcher); if ( hostAvailabilityListener != null ) { BatchFailureListener<QueryBatch> retryListener = hostAvailabilityListener.initializeRetryListener(this); if( retryListener != null ) onFailure(retryListener); } }
@Override public ExportToWriterListener withTransform(ServerTransform transform) { super.withTransform(transform); return this; }
@Override public boolean isComplete() { return totalResults > 0 ? queryBatch.getJobResultsSoFar() >= totalResults : false; }
/** * The format for the metadata retrieved with each document. The metadata will * be available in each DocumentRecord sent to the OutputListener registered * with onGenerateOutput. * * @param nonDocumentFormat the format for the metadata * @return this instance (for method chaining) */ @Override public ExportToWriterListener withNonDocumentFormat(Format nonDocumentFormat) { super.withNonDocumentFormat(nonDocumentFormat); return this; }
/** * A subclass can override this to provide a different implementation of ProgressUpdate. * * @param batch * @param startTime * @param totalForThisUpdate * @param timeSoFar * @return */ protected ProgressUpdate newProgressUpdate(QueryBatch batch, long startTime, long totalForThisUpdate, double timeSoFar) { return new SimpleProgressUpdate(batch, startTime, totalForThisUpdate, timeSoFar); }