/** * Creates a new instance of the Consumer class. * * @param streamName The name of the Stream to monitor. * @param config Test Configuration. * @param dataSource Data Source. * @param testState A TestState representing the current state of the test. This will be used for reporting purposes. * @param store A StoreAdapter to execute operations on. * @param executorService The Executor Service to use for async tasks. */ Consumer(String streamName, TestConfig config, ProducerDataSource dataSource, TestState testState, StoreAdapter store, ScheduledExecutorService executorService) { super(config, dataSource, store, executorService); Preconditions.checkArgument(canUseStoreAdapter(store), "StoreAdapter does not support all required features; cannot create a consumer for it."); this.logId = String.format("Consumer[%s]", streamName); this.streamName = Preconditions.checkNotNull(streamName, "streamName"); this.testState = Preconditions.checkNotNull(testState, "testState"); this.reader = store.createReader(); this.catchupReadsSupported = store.isFeatureSupported(StoreAdapter.Feature.RandomRead); this.storageReadsSupported = store.isFeatureSupported(StoreAdapter.Feature.StorageDirect); this.cancellationToken = new CancellationToken(); this.lastSequenceNumbers = new ConcurrentHashMap<>(); this.catchupQueue = new BlockingDrainingQueue<>(); this.storageQueue = new ArrayDeque<>(); this.storageReadQueue = new ArrayDeque<>(); }
/** * Creates a new instance of the OperationProcessor class. * * @param metadata The ContainerMetadata for the Container to process operations for. * @param stateUpdater A MemoryStateUpdater that is used to update in-memory structures upon successful Operation committal. * @param durableDataLog The DataFrameLog to write DataFrames to. * @param checkpointPolicy The Checkpoint Policy for Metadata. * @param executor An Executor to use for async operations. * @throws NullPointerException If any of the arguments are null. */ OperationProcessor(UpdateableContainerMetadata metadata, MemoryStateUpdater stateUpdater, DurableDataLog durableDataLog, MetadataCheckpointPolicy checkpointPolicy, ScheduledExecutorService executor) { super(String.format("OperationProcessor[%d]", metadata.getContainerId()), executor); Preconditions.checkNotNull(durableDataLog, "durableDataLog"); this.metadata = metadata; this.stateUpdater = Preconditions.checkNotNull(stateUpdater, "stateUpdater"); this.metadataUpdater = new OperationMetadataUpdater(this.metadata); this.operationQueue = new BlockingDrainingQueue<>(); this.commitQueue = new BlockingDrainingQueue<>(); this.state = new QueueProcessingState(checkpointPolicy); val args = new DataFrameBuilder.Args(this.state::frameSealed, this.state::commit, this.state::fail, this.executor); this.dataFrameBuilder = new DataFrameBuilder<>(durableDataLog, OperationSerializer.DEFAULT, args); this.metrics = new SegmentStoreMetrics.OperationProcessor(this.metadata.getContainerId()); this.throttlerCalculator = ThrottlerCalculator.builder() .cacheThrottler(stateUpdater::getCacheUtilization) .commitBacklogThrottler(this.commitQueue::size) .batchingThrottler(durableDataLog::getQueueStatistics) .build(); }
/** * Tests the ability of the queue to cancel a take() request if it is closed. */ @Test public void testCloseCancel() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); CompletableFuture<Queue<Integer>> result = queue.take(MAX_READ_COUNT); Collection<Integer> queueContents = queue.close(); // Verify result. AssertExtensions.assertThrows( "Future was not cancelled with the correct exception.", () -> result.get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), ex -> ex instanceof CancellationException); Assert.assertEquals("Queue.close() returned an item even though it was empty.", 0, queueContents.size()); }
/** * Tests the basic ability to dequeue items as a batch using poll(). */ @Test public void testAddMultiPoll() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); populate(queue); for (int i = 0; i < ITEM_COUNT; i += MAX_READ_COUNT) { Queue<Integer> entries = queue.poll(MAX_READ_COUNT); int expectedCount = Math.min(MAX_READ_COUNT, ITEM_COUNT - i); Assert.assertEquals("Unexpected number of items polled.", expectedCount, entries.size()); int expectedValue = i; for (int value : entries) { Assert.assertEquals("Unexpected value polled from queue.", expectedValue, value); expectedValue++; } } }
/** * Tests the basic ability dequeue items using poll() as they are added. */ @Test public void testAddSinglePoll() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); for (int i = 0; i < ITEM_COUNT; i++) { queue.add(i); Queue<Integer> entries = queue.poll(MAX_READ_COUNT); Assert.assertEquals("Unexpected number of items polled.", 1, entries.size()); Assert.assertEquals("Unexpected value polled from queue.", i, (int) entries.peek()); } val remainingItems = queue.poll(1); Assert.assertEquals("poll() did not return an empty collection when queue was empty.", 0, remainingItems.size()); }
/** * Tests the basic ability to dequeue items using take() as they are added. */ @Test public void testAddSingleTake() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); for (int i = 0; i < ITEM_COUNT; i++) { queue.add(i); val takeResult = queue.take(MAX_READ_COUNT); Assert.assertTrue("take() returned an incomplete Future when data is available.", Futures.isSuccessful(takeResult)); val entries = takeResult.join(); Assert.assertEquals("Unexpected number of items polled.", 1, entries.size()); Assert.assertEquals("Unexpected value polled from queue.", i, (int) entries.peek()); } val remainingItems = queue.take(1); Assert.assertFalse("take() did not return an incomplete future when queue was empty.", remainingItems.isDone()); }
/** * Tests the ability of the queue to return its contents when it is closed. */ @Test public void testCloseResult() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); populate(queue); Collection<Integer> queueContents = queue.close(); // Verify result. Assert.assertEquals("Unexpected result size from Queue.close().", ITEM_COUNT, queueContents.size()); int expectedValue = 0; for (int value : queueContents) { Assert.assertEquals("Unexpected value in Queue result.", expectedValue, value); expectedValue++; } }
/** * Tests the basic ability to dequeue items as a batch using take(). */ @Test public void testAddMultiTake() throws Exception { @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); populate(queue); for (int i = 0; i < ITEM_COUNT; i += MAX_READ_COUNT) { val takeResult = queue.take(MAX_READ_COUNT); Assert.assertTrue("take() returned an incomplete Future when data is available.", Futures.isSuccessful(takeResult)); val entries = takeResult.join(); int expectedCount = Math.min(MAX_READ_COUNT, ITEM_COUNT - i); Assert.assertEquals("Unexpected number of items polled.", expectedCount, entries.size()); int expectedValue = i; for (int value : entries) { Assert.assertEquals("Unexpected value polled from queue.", expectedValue, value); expectedValue++; } } }
/** * Tests the ability to cancel a pending take() operation. */ @Test public void testCancelPendingTake() throws Exception { final int valueToQueue = 1234; @Cleanup BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); val takeResult = queue.take(MAX_READ_COUNT); Assert.assertFalse("take() returned a completed future.", takeResult.isDone()); queue.cancelPendingTake(); Assert.assertTrue("cancelPendingTake() did not cancel a pending take() future.", takeResult.isCancelled()); val takeResult2 = queue.take(MAX_READ_COUNT); queue.add(valueToQueue); Assert.assertEquals("take() did not work again after being cancelled.", valueToQueue, (int) takeResult2.get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS).poll()); }
BlockingDrainingQueue<Integer> queue = new BlockingDrainingQueue<>(); val takeResult = queue.take(MAX_READ_COUNT);