private <K, V> Fetcher<K, V> createFetcher(SubscriptionState subscriptions, Metrics metrics, Deserializer<K> keyDeserializer, Deserializer<V> valueDeserializer) { return createFetcher(subscriptions, metrics, keyDeserializer, valueDeserializer, Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); }
private Fetcher<byte[], byte[]> createFetcher(SubscriptionState subscriptions, Metrics metrics, int maxPollRecords) { return createFetcher(subscriptions, metrics, new ByteArrayDeserializer(), new ByteArrayDeserializer(), maxPollRecords, IsolationLevel.READ_UNCOMMITTED); }
@Test public void testReadCommittedWithCompactedTopic() { Fetcher<String, String> fetcher = createFetcher(subscriptions, new Metrics(), new StringDeserializer(), new StringDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024);
@Test public void testMultipleAbortMarkers() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024);
@Test public void testHeaders() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(time));
@Test public void testReturnAbortedTransactionsinUncommittedMode() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024); int currentOffset = 0; currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); abortTransaction(buffer, 1L, currentOffset); buffer.flip(); List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>(); abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); // normal fetch assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords(); assertTrue(fetchedRecords.containsKey(tp0)); }
@Test public void testReadCommittedAbortMarkerWithNoData() { Fetcher<String, String> fetcher = createFetcher(subscriptions, new Metrics(), new StringDeserializer(), new StringDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024); long producerId = 1L; abortTransaction(buffer, producerId, 5L); appendTransactionalRecords(buffer, producerId, 6L, new SimpleRecord("6".getBytes(), null), new SimpleRecord("7".getBytes(), null), new SimpleRecord("8".getBytes(), null)); commitTransaction(buffer, producerId, 9L); buffer.flip(); // send the fetch subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); // prepare the response. the aborted transactions begin at offsets which are no longer in the log List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>(); abortedTransactions.add(new FetchResponse.AbortedTransaction(producerId, 0L)); client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map<TopicPartition, List<ConsumerRecord<String, String>>> allFetchedRecords = fetcher.fetchedRecords(); assertTrue(allFetchedRecords.containsKey(tp0)); List<ConsumerRecord<String, String>> fetchedRecords = allFetchedRecords.get(tp0); assertEquals(3, fetchedRecords.size()); assertEquals(Arrays.asList(6L, 7L, 8L), collectRecordOffsets(fetchedRecords)); }
@Test public void testSkippingAbortedTransactions() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024); int currentOffset = 0; currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); abortTransaction(buffer, 1L, currentOffset); buffer.flip(); List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>(); abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); // normal fetch assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords(); assertFalse(fetchedRecords.containsKey(tp0)); }
@Test public void testSeekBeforeException() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptionsNoAutoReset, new Metrics(time), 2); subscriptionsNoAutoReset.assignFromUser(Utils.mkSet(tp0)); subscriptionsNoAutoReset.seek(tp0, 1); assertEquals(1, fetcher.sendFetches()); Map<TopicPartition, FetchResponse.PartitionData<MemoryRecords>> partitions = new HashMap<>(); partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, records)); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEquals(2, fetcher.fetchedRecords().get(tp0).size()); subscriptionsNoAutoReset.assignFromUser(Utils.mkSet(tp0, tp1)); subscriptionsNoAutoReset.seek(tp1, 1); assertEquals(1, fetcher.sendFetches()); partitions = new HashMap<>(); partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)); client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), 0, INVALID_SESSION_ID)); consumerClient.poll(time.timer(0)); assertEquals(1, fetcher.fetchedRecords().get(tp0).size()); subscriptionsNoAutoReset.seek(tp1, 10); // Should not throw OffsetOutOfRangeException after the seek assertEquals(0, fetcher.fetchedRecords().size()); }
@Test public void testReturnCommittedTransactions() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024);
@Test public void testReadCommittedWithCommittedAndAbortedTransactions() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024);
@Test public void testFetcherMetricsTemplates() throws Exception { metrics.close(); Map<String, String> clientTags = Collections.singletonMap("client-id", "clientA"); metrics = new Metrics(new MetricConfig().tags(clientTags)); metricsRegistry = new FetcherMetricsRegistry(clientTags.keySet(), "consumer" + groupId); fetcher.close(); fetcher = createFetcher(subscriptions, metrics); // Fetch from topic to generate topic metrics subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetcher.fetchedRecords(); assertTrue(partitionRecords.containsKey(tp0)); // Create throttle metrics Fetcher.throttleTimeSensor(metrics, metricsRegistry); // Verify that all metrics except metrics-count have registered templates Set<MetricNameTemplate> allMetrics = new HashSet<>(); for (MetricName n : metrics.metrics().keySet()) { String name = n.name().replaceAll(tp0.toString(), "{topic}-{partition}"); if (!n.group().equals("kafka-metrics-count")) allMetrics.add(new MetricNameTemplate(name, n.group(), "", n.tags().keySet())); } TestUtils.checkEquals(allMetrics, new HashSet<>(metricsRegistry.getAllTemplates()), "metrics", "templates"); }
@Test public void testReadCommittedLagMetric() { Metrics metrics = new Metrics(); fetcher = createFetcher(subscriptions, metrics, new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
@Test public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024); long currentOffset = 0; currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()), new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes())); currentOffset += abortTransaction(buffer, 1L, currentOffset); buffer.flip(); List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>(); abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); // normal fetch assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords(); // Ensure that we don't return any of the aborted records, but yet advance the consumer position. assertFalse(fetchedRecords.containsKey(tp0)); assertEquals(currentOffset, (long) subscriptions.position(tp0)); }
@Test public void testEmptyControlBatch() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024);
@Test public void testFetchMaxPollRecords() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(time), 2); List<ConsumerRecord<byte[], byte[]>> records; subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); client.prepareResponse(matchesOffset(tp0, 1), fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); client.prepareResponse(matchesOffset(tp0, 4), fullFetchResponse(tp0, this.nextRecords, Errors.NONE, 100L, 0)); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(time.timer(0)); records = fetcher.fetchedRecords().get(tp0); assertEquals(2, records.size()); assertEquals(3L, subscriptions.position(tp0).longValue()); assertEquals(1, records.get(0).offset()); assertEquals(2, records.get(1).offset()); assertEquals(0, fetcher.sendFetches()); consumerClient.poll(time.timer(0)); records = fetcher.fetchedRecords().get(tp0); assertEquals(1, records.size()); assertEquals(4L, subscriptions.position(tp0).longValue()); assertEquals(3, records.get(0).offset()); assertTrue(fetcher.sendFetches() > 0); consumerClient.poll(time.timer(0)); records = fetcher.fetchedRecords().get(tp0); assertEquals(2, records.size()); assertEquals(6L, subscriptions.position(tp0).longValue()); assertEquals(4, records.get(0).offset()); assertEquals(5, records.get(1).offset()); }
Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(time), 2);
@Test public void testConsumingViaIncrementalFetchRequests() { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(time), 2);
@Test public void testListOffsetsSendsIsolationLevel() { for (final IsolationLevel isolationLevel : IsolationLevel.values()) { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, isolationLevel); subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { ListOffsetRequest request = (ListOffsetRequest) body; return request.isolationLevel() == isolationLevel; } }, listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); } }
Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(time), deserializer, deserializer);