} else if (!subscriptions.isFetchable(partitionRecords.partition)) {
@Test public void partitionPause() { state.assignFromUser(singleton(tp0)); state.seek(tp0, 100); assertTrue(state.isFetchable(tp0)); state.pause(tp0); assertFalse(state.isFetchable(tp0)); state.resume(tp0); assertTrue(state.isFetchable(tp0)); }
if (!subscriptions.isFetchable(tp)) {
@Test public void testUpdateFetchPositionNoOpWithPositionSet() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 5L); fetcher.resetOffsetsIfNeeded(); assertFalse(client.hasInFlightRequests()); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void partitionAssignment() { state.assignFromUser(singleton(tp0)); assertEquals(singleton(tp0), state.assignedPartitions()); assertEquals(1, state.numAssignedPartitions()); assertFalse(state.hasAllFetchPositions()); state.seek(tp0, 1); assertTrue(state.isFetchable(tp0)); assertEquals(1L, state.position(tp0).longValue()); state.assignFromUser(Collections.<TopicPartition>emptySet()); assertTrue(state.assignedPartitions().isEmpty()); assertEquals(0, state.numAssignedPartitions()); assertFalse(state.isAssigned(tp0)); assertFalse(state.isFetchable(tp0)); }
@Test public void testUpdateFetchPositionOfPausedPartitionsWithAValidPosition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 10); subscriptions.pause(tp0); // paused partition already has a valid position fetcher.resetOffsetsIfNeeded(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertTrue(subscriptions.hasValidPosition(tp0)); assertEquals(10, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionResetToLatestOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void partitionReset() { state.assignFromUser(singleton(tp0)); state.seek(tp0, 5); assertEquals(5L, (long) state.position(tp0)); state.requestOffsetReset(tp0); assertFalse(state.isFetchable(tp0)); assertTrue(state.isOffsetResetNeeded(tp0)); assertEquals(null, state.position(tp0)); // seek should clear the reset and make the partition fetchable state.seek(tp0, 0); assertTrue(state.isFetchable(tp0)); assertFalse(state.isOffsetResetNeeded(tp0)); }
@Test public void testUpdateFetchPositionResetToDefaultOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testFetchDisconnected() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0), true); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); // disconnects should have no affect on subscription state assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(0, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionResetToEarliestOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void topicSubscription() { state.subscribe(singleton(topic), rebalanceListener); assertEquals(1, state.subscription().size()); assertTrue(state.assignedPartitions().isEmpty()); assertEquals(0, state.numAssignedPartitions()); assertTrue(state.partitionsAutoAssigned()); state.assignFromSubscribed(singleton(tp0)); state.seek(tp0, 1); assertEquals(1L, state.position(tp0).longValue()); state.assignFromSubscribed(singleton(tp1)); assertTrue(state.isAssigned(tp1)); assertFalse(state.isAssigned(tp0)); assertFalse(state.isFetchable(tp1)); assertEquals(singleton(tp1), state.assignedPartitions()); assertEquals(1, state.numAssignedPartitions()); }
@Test public void testUpdateFetchPositionOfPausedPartitionsWithoutAValidPosition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0); subscriptions.pause(tp0); // paused partition does not have a valid position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertFalse(subscriptions.hasValidPosition(tp0)); }
@Test public void testUpdateFetchPositionOfPausedPartitionsRequiringOffsetReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.pause(tp0); // paused partition does not have a valid position subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 10L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertTrue(subscriptions.hasValidPosition(tp0)); assertEquals(10, subscriptions.position(tp0).longValue()); }
@Test public void testListOffsetsSendsIsolationLevel() { for (final IsolationLevel isolationLevel : IsolationLevel.values()) { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, isolationLevel); subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { ListOffsetRequest request = (ListOffsetRequest) body; return request.isolationLevel() == isolationLevel; } }, listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); } }
assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); assertTrue(subscriptions.hasValidPosition(tp0)); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(subscriptions.position(tp0).longValue(), 5L);
@Test public void testResetOffsetsSkipsBlackedOutConnections() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); // Check that we skip sending the ListOffset request when the node is blacked out client.blackout(node, 500); fetcher.resetOffsetsIfNeeded(); assertEquals(0, consumerClient.pendingRequestCount()); consumerClient.pollNoWakeup(); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(OffsetResetStrategy.EARLIEST, subscriptions.resetStrategy(tp0)); time.sleep(500); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testResetOffsetsMetadataRefresh() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // First fetch fails with stale metadata client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NOT_LEADER_FOR_PARTITION, 1L, 5L), false); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); // Expect a metadata refresh client.prepareMetadataUpdate(initialUpdateResponse); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingMetadataUpdates()); // Next fetch succeeds time.sleep(retryBackoffMs); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue());
@Test public void testUpdateFetchPositionDisconnect() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // First request gets a disconnect client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L), true); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); // Expect a metadata refresh client.prepareMetadataUpdate(initialUpdateResponse); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingMetadataUpdates()); // No retry until the backoff passes fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(client.hasInFlightRequests()); assertFalse(subscriptions.hasValidPosition(tp0)); // Next one succeeds time.sleep(retryBackoffMs); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }