private void populate(BlockingDrainingQueue<Integer> queue) { for (int i = 0; i < ITEM_COUNT; i++) { queue.add(i); } } }
@Override protected CompletableFuture<Void> doRun() { // The QueueProcessor is responsible with the processing of externally added Operations. It starts when the // OperationProcessor starts and is shut down as soon as doStop() is invoked. val queueProcessor = Futures .loop(this::isRunning, () -> throttle() .thenComposeAsync(v -> this.operationQueue.take(MAX_READ_AT_ONCE), this.executor) .thenAcceptAsync(this::processOperations, this.executor), this.executor); // The CommitProcessor is responsible with the processing of those Operations that have already been committed to // DurableDataLong and now need to be added to the in-memory State. // As opposed from the QueueProcessor, this needs to process all pending commits and not discard them, even when // we receive a stop signal (from doStop()), otherwise we could be left with an inconsistent in-memory state. val commitProcessor = Futures .loop(() -> isRunning() || this.commitQueue.size() > 0, () -> this.commitQueue.take(MAX_COMMIT_QUEUE_SIZE) .thenAcceptAsync(this::processCommits, this.executor), this.executor) .whenComplete((r, ex) -> { // The CommitProcessor is done. Safe to close its queue now, regardless of whether it failed or // shut down normally. this.commitQueue.close(); if (ex != null) { throw new CompletionException(ex); } }); return CompletableFuture.allOf(queueProcessor, commitProcessor) .exceptionally(this::iterationErrorHandler); }
/** * Closes the Operation Queue and fails all Operations in it with the given exception. * * @param causingException The exception to fail with. If null, it will default to ObjectClosedException. */ private void closeQueue(Throwable causingException) { // Close the operation queue and extract any outstanding Operations from it. Collection<CompletableOperation> remainingOperations = this.operationQueue.close(); if (remainingOperations != null && remainingOperations.size() > 0) { // If any outstanding Operations were left in the queue, they need to be failed. // If no other cause was passed, assume we are closing the queue because we are shutting down. Throwable failException = causingException != null ? causingException : new CancellationException(); cancelIncompleteOperations(remainingOperations, failException); } // The commit queue will auto-close when we are done and it itself is empty. We just need to unblock it in case // it was idle and waiting on a pending take() operation. this.commitQueue.cancelPendingTake(); }
/** * Tests the ability to cancel a pending take() operation. */ @Test public void testCancelPendingTake() throws Exception { final int valueToQueue = 1234; @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); val takeResult = queue.take(MAX_READ_COUNT); Assert.assertFalse("take() returned a completed future.", takeResult.isDone()); queue.cancelPendingTake(); Assert.assertTrue("cancelPendingTake() did not cancel a pending take() future.", takeResult.isCancelled()); val takeResult2 = queue.take(MAX_READ_COUNT); queue.add(valueToQueue); Assert.assertEquals("take() did not work again after being cancelled.", valueToQueue, (int) takeResult2.get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS).poll()); }
BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); val takeResult = queue.take(MAX_READ_COUNT); () -> queue.take(MAX_READ_COUNT), ex -> ex instanceof IllegalStateException); () -> queue.poll(MAX_READ_COUNT), ex -> ex instanceof IllegalStateException); queue.add(valueToQueue); Assert.assertEquals("Unexpected value polled from queue.", valueToQueue, (int) result.peek()); val remainingItems = queue.poll(MAX_READ_COUNT); Assert.assertEquals("Queue was not emptied out after take() completed successfully.", 0, remainingItems.size());
/** * Tests the ability of the queue to cancel a take() request if it is closed. */ @Test public void testCloseCancel() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); CompletableFuture<Queue<Integer>> result = queue.take(MAX_READ_COUNT); Collection<Integer> queueContents = queue.close(); // Verify result. AssertExtensions.assertThrows( "Future was not cancelled with the correct exception.", () -> result.get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), ex -> ex instanceof CancellationException); Assert.assertEquals("Queue.close() returned an item even though it was empty.", 0, queueContents.size()); }
/** * Tests the basic ability to dequeue items using take() as they are added. */ @Test public void testAddSingleTake() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); for (int i = 0; i < ITEM_COUNT; i++) { queue.add(i); val takeResult = queue.take(MAX_READ_COUNT); Assert.assertTrue("take() returned an incomplete Future when data is available.", Futures.isSuccessful(takeResult)); val entries = takeResult.join(); Assert.assertEquals("Unexpected number of items polled.", 1, entries.size()); Assert.assertEquals("Unexpected value polled from queue.", i, (int) entries.peek()); } val remainingItems = queue.take(1); Assert.assertFalse("take() did not return an incomplete future when queue was empty.", remainingItems.isDone()); }
/** * Tests the basic ability dequeue items using poll() as they are added. */ @Test public void testAddSinglePoll() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); for (int i = 0; i < ITEM_COUNT; i++) { queue.add(i); Queue<Integer> entries = queue.poll(MAX_READ_COUNT); Assert.assertEquals("Unexpected number of items polled.", 1, entries.size()); Assert.assertEquals("Unexpected value polled from queue.", i, (int) entries.peek()); } val remainingItems = queue.poll(1); Assert.assertEquals("poll() did not return an empty collection when queue was empty.", 0, remainingItems.size()); }
/** * Tests the ability of the queue to return its contents when it is closed. */ @Test public void testCloseResult() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); populate(queue); Collection<Integer> queueContents = queue.close(); // Verify result. Assert.assertEquals("Unexpected result size from Queue.close().", ITEM_COUNT, queueContents.size()); int expectedValue = 0; for (int value : queueContents) { Assert.assertEquals("Unexpected value in Queue result.", expectedValue, value); expectedValue++; } }
/** * Tests the basic ability to dequeue items as a batch using take(). */ @Test public void testAddMultiTake() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); populate(queue); for (int i = 0; i < ITEM_COUNT; i += MAX_READ_COUNT) { val takeResult = queue.take(MAX_READ_COUNT); Assert.assertTrue("take() returned an incomplete Future when data is available.", Futures.isSuccessful(takeResult)); val entries = takeResult.join(); int expectedCount = Math.min(MAX_READ_COUNT, ITEM_COUNT - i); Assert.assertEquals("Unexpected number of items polled.", expectedCount, entries.size()); int expectedValue = i; for (int value : entries) { Assert.assertEquals("Unexpected value polled from queue.", expectedValue, value); expectedValue++; } } }
/** * Tests the basic ability to dequeue items as a batch using poll(). */ @Test public void testAddMultiPoll() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); populate(queue); for (int i = 0; i < ITEM_COUNT; i += MAX_READ_COUNT) { Queue<Integer> entries = queue.poll(MAX_READ_COUNT); int expectedCount = Math.min(MAX_READ_COUNT, ITEM_COUNT - i); Assert.assertEquals("Unexpected number of items polled.", expectedCount, entries.size()); int expectedValue = i; for (int value : entries) { Assert.assertEquals("Unexpected value polled from queue.", expectedValue, value); expectedValue++; } } }
/** * Creates a new instance of the Consumer class. * * @param streamName The name of the Stream to monitor. * @param config Test Configuration. * @param dataSource Data Source. * @param testState A TestState representing the current state of the test. This will be used for reporting purposes. * @param store A StoreAdapter to execute operations on. * @param executorService The Executor Service to use for async tasks. */ Consumer(String streamName, TestConfig config, ProducerDataSource dataSource, TestState testState, StoreAdapter store, ScheduledExecutorService executorService) { super(config, dataSource, store, executorService); Preconditions.checkArgument(canUseStoreAdapter(store), "StoreAdapter does not support all required features; cannot create a consumer for it."); this.logId = String.format("Consumer[%s]", streamName); this.streamName = Preconditions.checkNotNull(streamName, "streamName"); this.testState = Preconditions.checkNotNull(testState, "testState"); this.reader = store.createReader(); this.catchupReadsSupported = store.isFeatureSupported(StoreAdapter.Feature.RandomRead); this.storageReadsSupported = store.isFeatureSupported(StoreAdapter.Feature.StorageDirect); this.cancellationToken = new CancellationToken(); this.lastSequenceNumbers = new ConcurrentHashMap<>(); this.catchupQueue = new BlockingDrainingQueue<>(); this.storageQueue = new ArrayDeque<>(); this.storageReadQueue = new ArrayDeque<>(); }
private CompletableFuture<Void> processCatchupReads() { if (!this.catchupReadsSupported) { return CompletableFuture.completedFuture(null); } return Futures.loop( this::canRun, () -> this.catchupQueue.take(CATCHUP_READ_COUNT) .thenComposeAsync(this::processCatchupReads, this.executorService), this.executorService) .exceptionally(ex -> { ex = Exceptions.unwrap(ex); if (ex instanceof ObjectClosedException) { // This a normal shutdown, as the catchupQueue is closed when we are done. return null; } throw new CompletionException(ex); }); }
this.metrics.currentState(this.operationQueue.size() + count, this.state.getPendingCount()); this.metrics.processOperations(count, processTimer.getElapsedMillis()); processTimer = new Timer(); // Reset this timer since we may be pulling in new operations. operations = this.operationQueue.poll(MAX_READ_AT_ONCE);
private CompletableFuture<Void> processCatchupRead(StoreReader.ReadItem toValidate) { if (toValidate instanceof EndItem) { this.catchupQueue.close(); return CompletableFuture.completedFuture(null); } final Timer timer = new Timer(); return this.reader .readExact(this.streamName, toValidate.getAddress()) .handleAsync((actualRead, ex) -> { ValidationResult validationResult; try { if (ex == null) { validationResult = compareReads(toValidate, actualRead.getEvent()); Event e = toValidate.getEvent(); this.testState.recordDuration(ConsumerOperationType.CATCHUP_READ, timer.getElapsed().toMillis()); this.testState.recordCatchupRead(e.getTotalLength()); } else { validationResult = ValidationResult.failed(ex.getMessage()); } } catch (Throwable ex2) { validationResult = ValidationResult.failed(String.format("General failure: Ex = %s.", ex2)); } if (!validationResult.isSuccess()) { validationResult.setAddress(toValidate.getAddress()); validationFailed(ValidationSource.CatchupRead, validationResult); } return null; }, this.executorService); }
private void processCommits(Collection<List<CompletableOperation>> items) { try { do { this.stateUpdater.process(items.stream().flatMap(List::stream).map(CompletableOperation::getOperation).iterator()); items = this.commitQueue.poll(MAX_COMMIT_QUEUE_SIZE); } while (!items.isEmpty()); } catch (Throwable ex) { // MemoryStateUpdater.process() should only throw DataCorruptionExceptions, but just in case it // throws something else (i.e. NullPtr), we still need to handle it. log.error("{}: MemoryStateUpdater.process failure.", traceObjectId, ex); // Then fail the remaining operations (which also handles fatal errors) and bail out. if (isFatalException(ex)) { Callbacks.invokeSafely(OperationProcessor.this::errorHandler, ex, null); } } }
/** * Closes the queue and prevents any other access to it. Any blocked call to takeAllItems() will fail with InterruptedException. * * @return If the queue has any more items in it, these will be returned here in the order in which they were inserted. * The items are guaranteed not to be returned both here and via take()/poll(). */ public Queue<T> close() { CompletableFuture<Queue<T>> pending = null; Queue<T> result = null; synchronized (this.contents) { if (!this.closed) { this.closed = true; pending = this.pendingTake; this.pendingTake = null; result = fetch(this.contents.size()); } } // Cancel any pending poll request. if (pending != null) { pending.cancel(true); } return result != null ? result : new LinkedList<>(); }
/** * Creates a new instance of the OperationProcessor class. * * @param metadata The ContainerMetadata for the Container to process operations for. * @param stateUpdater A MemoryStateUpdater that is used to update in-memory structures upon successful Operation committal. * @param durableDataLog The DataFrameLog to write DataFrames to. * @param checkpointPolicy The Checkpoint Policy for Metadata. * @param executor An Executor to use for async operations. * @throws NullPointerException If any of the arguments are null. */ OperationProcessor(UpdateableContainerMetadata metadata, MemoryStateUpdater stateUpdater, DurableDataLog durableDataLog, MetadataCheckpointPolicy checkpointPolicy, ScheduledExecutorService executor) { super(String.format("OperationProcessor[%d]", metadata.getContainerId()), executor); Preconditions.checkNotNull(durableDataLog, "durableDataLog"); this.metadata = metadata; this.stateUpdater = Preconditions.checkNotNull(stateUpdater, "stateUpdater"); this.metadataUpdater = new OperationMetadataUpdater(this.metadata); this.operationQueue = new BlockingDrainingQueue<>(); this.commitQueue = new BlockingDrainingQueue<>(); this.state = new QueueProcessingState(checkpointPolicy); val args = new DataFrameBuilder.Args(this.state::frameSealed, this.state::commit, this.state::fail, this.executor); this.dataFrameBuilder = new DataFrameBuilder<>(durableDataLog, OperationSerializer.DEFAULT, args); this.metrics = new SegmentStoreMetrics.OperationProcessor(this.metadata.getContainerId()); this.throttlerCalculator = ThrottlerCalculator.builder() .cacheThrottler(stateUpdater::getCacheUtilization) .commitBacklogThrottler(this.commitQueue::size) .batchingThrottler(durableDataLog::getQueueStatistics) .build(); }
/** * Returns the next items from the queue, if any. * * @param maxCount The maximum number of items to return. * @return A Queue containing at most maxCount items, or empty if there is nothing in the queue. * @throws IllegalStateException If there is a pending take() operation which hasn't completed yet. */ public Queue<T> poll(int maxCount) { synchronized (this.contents) { Exceptions.checkNotClosed(this.closed, this); Preconditions.checkState(this.pendingTake == null, "Cannot call poll() when there is a pending take() request."); return fetch(maxCount); } }
/** * Processes the given Operation. This method returns when the given Operation has been added to the internal queue. * * @param operation The Operation to process. * @return A CompletableFuture that, when completed, will indicate the Operation has finished processing. If the * Operation completed successfully, the Future will contain the Sequence Number of the Operation. If the Operation * failed, it will contain the exception that caused the failure. * @throws IllegalContainerStateException If the OperationProcessor is not running. */ public CompletableFuture<Void> process(Operation operation) { CompletableFuture<Void> result = new CompletableFuture<>(); if (!isRunning()) { result.completeExceptionally(new IllegalContainerStateException("OperationProcessor is not running.")); } else { log.debug("{}: process {}.", this.traceObjectId, operation); try { this.operationQueue.add(new CompletableOperation(operation, result)); } catch (Throwable e) { if (Exceptions.mustRethrow(e)) { throw e; } result.completeExceptionally(e); } } return result; }