private CompletableFuture<Void> processCatchupReads() { if (!this.catchupReadsSupported) { return CompletableFuture.completedFuture(null); } return Futures.loop( this::canRun, () -> this.catchupQueue.take(CATCHUP_READ_COUNT) .thenComposeAsync(this::processCatchupReads, this.executorService), this.executorService) .exceptionally(ex -> { ex = Exceptions.unwrap(ex); if (ex instanceof ObjectClosedException) { // This a normal shutdown, as the catchupQueue is closed when we are done. return null; } throw new CompletionException(ex); }); }
@Override protected CompletableFuture<Void> doRun() { // The QueueProcessor is responsible with the processing of externally added Operations. It starts when the // OperationProcessor starts and is shut down as soon as doStop() is invoked. val queueProcessor = Futures .loop(this::isRunning, () -> throttle() .thenComposeAsync(v -> this.operationQueue.take(MAX_READ_AT_ONCE), this.executor) .thenAcceptAsync(this::processOperations, this.executor), this.executor); // The CommitProcessor is responsible with the processing of those Operations that have already been committed to // DurableDataLong and now need to be added to the in-memory State. // As opposed from the QueueProcessor, this needs to process all pending commits and not discard them, even when // we receive a stop signal (from doStop()), otherwise we could be left with an inconsistent in-memory state. val commitProcessor = Futures .loop(() -> isRunning() || this.commitQueue.size() > 0, () -> this.commitQueue.take(MAX_COMMIT_QUEUE_SIZE) .thenAcceptAsync(this::processCommits, this.executor), this.executor) .whenComplete((r, ex) -> { // The CommitProcessor is done. Safe to close its queue now, regardless of whether it failed or // shut down normally. this.commitQueue.close(); if (ex != null) { throw new CompletionException(ex); } }); return CompletableFuture.allOf(queueProcessor, commitProcessor) .exceptionally(this::iterationErrorHandler); }
/** * Tests the basic ability to dequeue items using take() as they are added. */ @Test public void testAddSingleTake() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); for (int i = 0; i < ITEM_COUNT; i++) { queue.add(i); val takeResult = queue.take(MAX_READ_COUNT); Assert.assertTrue("take() returned an incomplete Future when data is available.", Futures.isSuccessful(takeResult)); val entries = takeResult.join(); Assert.assertEquals("Unexpected number of items polled.", 1, entries.size()); Assert.assertEquals("Unexpected value polled from queue.", i, (int) entries.peek()); } val remainingItems = queue.take(1); Assert.assertFalse("take() did not return an incomplete future when queue was empty.", remainingItems.isDone()); }
/** * Tests the ability to cancel a pending take() operation. */ @Test public void testCancelPendingTake() throws Exception { final int valueToQueue = 1234; @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); val takeResult = queue.take(MAX_READ_COUNT); Assert.assertFalse("take() returned a completed future.", takeResult.isDone()); queue.cancelPendingTake(); Assert.assertTrue("cancelPendingTake() did not cancel a pending take() future.", takeResult.isCancelled()); val takeResult2 = queue.take(MAX_READ_COUNT); queue.add(valueToQueue); Assert.assertEquals("take() did not work again after being cancelled.", valueToQueue, (int) takeResult2.get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS).poll()); }
/** * Tests the ability of the queue to cancel a take() request if it is closed. */ @Test public void testCloseCancel() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); CompletableFuture<Queue<Integer>> result = queue.take(MAX_READ_COUNT); Collection<Integer> queueContents = queue.close(); // Verify result. AssertExtensions.assertThrows( "Future was not cancelled with the correct exception.", () -> result.get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), ex -> ex instanceof CancellationException); Assert.assertEquals("Queue.close() returned an item even though it was empty.", 0, queueContents.size()); }
/** * Tests the basic ability to dequeue items as a batch using take(). */ @Test public void testAddMultiTake() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); populate(queue); for (int i = 0; i < ITEM_COUNT; i += MAX_READ_COUNT) { val takeResult = queue.take(MAX_READ_COUNT); Assert.assertTrue("take() returned an incomplete Future when data is available.", Futures.isSuccessful(takeResult)); val entries = takeResult.join(); int expectedCount = Math.min(MAX_READ_COUNT, ITEM_COUNT - i); Assert.assertEquals("Unexpected number of items polled.", expectedCount, entries.size()); int expectedValue = i; for (int value : entries) { Assert.assertEquals("Unexpected value polled from queue.", expectedValue, value); expectedValue++; } } }