public void requestOffsetReset(TopicPartition partition) { requestOffsetReset(partition, defaultResetStrategy); }
@Override public synchronized void seekToBeginning(Collection<TopicPartition> partitions) { ensureNotClosed(); for (TopicPartition tp : partitions) subscriptions.requestOffsetReset(tp, OffsetResetStrategy.EARLIEST); }
@Override public synchronized void seekToEnd(Collection<TopicPartition> partitions) { ensureNotClosed(); for (TopicPartition tp : partitions) subscriptions.requestOffsetReset(tp, OffsetResetStrategy.LATEST); }
/** * Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the * first offset in all partitions only when {@link #poll(Duration)} or {@link #position(TopicPartition)} are called. * If no partitions are provided, seek to the first offset for all of the currently assigned partitions. * * @throws IllegalArgumentException if {@code partitions} is {@code null} * @throws IllegalStateException if any of the provided partitions are not currently assigned to this consumer */ @Override public void seekToBeginning(Collection<TopicPartition> partitions) { if (partitions == null) throw new IllegalArgumentException("Partitions collection cannot be null"); acquireAndEnsureOpen(); try { Collection<TopicPartition> parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; for (TopicPartition tp : parts) { log.debug("Seeking to beginning of partition {}", tp); subscriptions.requestOffsetReset(tp, OffsetResetStrategy.EARLIEST); } } finally { release(); } }
/** * Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the * final offset in all partitions only when {@link #poll(Duration)} or {@link #position(TopicPartition)} are called. * If no partitions are provided, seek to the final offset for all of the currently assigned partitions. * <p> * If {@code isolation.level=read_committed}, the end offset will be the Last Stable Offset, i.e., the offset * of the first message with an open transaction. * * @throws IllegalArgumentException if {@code partitions} is {@code null} * @throws IllegalStateException if any of the provided partitions are not currently assigned to this consumer */ @Override public void seekToEnd(Collection<TopicPartition> partitions) { if (partitions == null) throw new IllegalArgumentException("Partitions collection cannot be null"); acquireAndEnsureOpen(); try { Collection<TopicPartition> parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; for (TopicPartition tp : parts) { log.debug("Seeking to end of partition {}", tp); subscriptions.requestOffsetReset(tp, OffsetResetStrategy.LATEST); } } finally { release(); } }
private void updateFetchPosition(TopicPartition tp) { if (subscriptions.isOffsetResetNeeded(tp)) { resetOffsetPosition(tp); } else if (!committed.containsKey(tp)) { subscriptions.requestOffsetReset(tp); resetOffsetPosition(tp); } else { subscriptions.seek(tp, committed.get(tp).offset()); } }
@Test public void testUpdateFetchPositionResetToLatestOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionResetToEarliestOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testNoCoordinatorDiscoveryIfPartitionAwaitingReset() { assertTrue(coordinator.coordinatorUnknown()); subscriptions.assignFromUser(singleton(t1p)); subscriptions.requestOffsetReset(t1p, OffsetResetStrategy.EARLIEST); coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.missingFetchPositions()); assertFalse(subscriptions.hasAllFetchPositions()); assertEquals(Collections.singleton(t1p), subscriptions.partitionsNeedingReset(time.milliseconds())); assertEquals(OffsetResetStrategy.EARLIEST, subscriptions.resetStrategy(t1p)); assertTrue(coordinator.coordinatorUnknown()); }
@Test public void testIdempotentResetWithInFlightReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // Send the ListOffsets request to reset the position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(client.hasInFlightRequests()); // Now we get a seek from the user subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.respond(listOffsetResponse(Errors.NONE, 1L, 5L)); consumerClient.pollNoWakeup(); assertFalse(client.hasInFlightRequests()); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(5L, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionResetToDefaultOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionOfPausedPartitionsWithoutAValidPosition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0); subscriptions.pause(tp0); // paused partition does not have a valid position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertFalse(subscriptions.hasValidPosition(tp0)); }
@Test public void testUpdateFetchPositionOfPausedPartitionsRequiringOffsetReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.pause(tp0); // paused partition does not have a valid position subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 10L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertTrue(subscriptions.hasValidPosition(tp0)); assertEquals(10, subscriptions.position(tp0).longValue()); }
@Test public void testChangeResetWithInFlightReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // Send the ListOffsets request to reset the position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(client.hasInFlightRequests()); // Now we get a seek from the user subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); // The response returns and is discarded client.respond(listOffsetResponse(Errors.NONE, 1L, 5L)); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingResponses()); assertFalse(client.hasInFlightRequests()); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(OffsetResetStrategy.EARLIEST, subscriptions.resetStrategy(tp0)); }
@Test public void testListOffsetsSendsIsolationLevel() { for (final IsolationLevel isolationLevel : IsolationLevel.values()) { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, isolationLevel); subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { ListOffsetRequest request = (ListOffsetRequest) body; return request.isolationLevel() == isolationLevel; } }, listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); } }
@Test public void testAssignmentChangeWithInFlightReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // Send the ListOffsets request to reset the position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(client.hasInFlightRequests()); // Now we have an assignment change subscriptions.assignFromUser(singleton(tp1)); // The response returns and is discarded client.respond(listOffsetResponse(Errors.NONE, 1L, 5L)); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingResponses()); assertFalse(client.hasInFlightRequests()); assertFalse(subscriptions.isAssigned(tp0)); }
@Test public void testSeekWithInFlightReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // Send the ListOffsets request to reset the position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(client.hasInFlightRequests()); // Now we get a seek from the user subscriptions.seek(tp0, 237); // The response returns and is discarded client.respond(listOffsetResponse(Errors.NONE, 1L, 5L)); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingResponses()); assertFalse(client.hasInFlightRequests()); assertEquals(237L, subscriptions.position(tp0).longValue()); }
@Test public void partitionReset() { state.assignFromUser(singleton(tp0)); state.seek(tp0, 5); assertEquals(5L, (long) state.position(tp0)); state.requestOffsetReset(tp0); assertFalse(state.isFetchable(tp0)); assertTrue(state.isOffsetResetNeeded(tp0)); assertEquals(null, state.position(tp0)); // seek should clear the reset and make the partition fetchable state.seek(tp0, 0); assertTrue(state.isFetchable(tp0)); assertFalse(state.isOffsetResetNeeded(tp0)); }
@Test public void testResetOffsetsSkipsBlackedOutConnections() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); // Check that we skip sending the ListOffset request when the node is blacked out client.blackout(node, 500); fetcher.resetOffsetsIfNeeded(); assertEquals(0, consumerClient.pendingRequestCount()); consumerClient.pollNoWakeup(); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(OffsetResetStrategy.EARLIEST, subscriptions.resetStrategy(tp0)); time.sleep(500); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testResetOffsetsMetadataRefresh() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // First fetch fails with stale metadata client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NOT_LEADER_FOR_PARTITION, 1L, 5L), false); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); // Expect a metadata refresh client.prepareMetadataUpdate(initialUpdateResponse); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingMetadataUpdates()); // Next fetch succeeds time.sleep(retryBackoffMs); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }