@Override public void onSuccess(Void value) { pendingAsyncCommits.decrementAndGet(); doCommitOffsetsAsync(offsets, callback); client.pollNoWakeup(); }
@Override public void run() { consumerClient.pollNoWakeup(); } };
/** * Leave the current group and reset local generation/memberId. */ public synchronized void maybeLeaveGroup() { if (!coordinatorUnknown() && state != MemberState.UNJOINED && generation.isValid()) { // this is a minimal effort attempt to leave the group. we do not // attempt any resending if the request fails or times out. log.info("Sending LeaveGroup request to coordinator {}", coordinator); LeaveGroupRequest.Builder request = new LeaveGroupRequest.Builder(groupId, generation.memberId); client.send(coordinator, request) .compose(new LeaveGroupResponseHandler()); client.pollNoWakeup(); } resetGeneration(); }
@Test public void testManyInFlightAsyncCommitsWithCoordinatorDisconnect() throws Exception { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); int numRequests = 1000; TopicPartition tp = new TopicPartition("foo", 0); final AtomicInteger responses = new AtomicInteger(0); for (int i = 0; i < numRequests; i++) { Map<TopicPartition, OffsetAndMetadata> offsets = singletonMap(tp, new OffsetAndMetadata(i)); coordinator.commitOffsetsAsync(offsets, new OffsetCommitCallback() { @Override public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) { responses.incrementAndGet(); Throwable cause = exception.getCause(); assertTrue("Unexpected exception cause type: " + (cause == null ? null : cause.getClass()), cause instanceof DisconnectException); } }); } coordinator.markCoordinatorUnknown(); consumerClient.pollNoWakeup(); coordinator.invokeCompletedOffsetCommitCallbacks(); assertEquals(numRequests, responses.get()); }
private void testInFlightRequestsFailedAfterCoordinatorMarkedDead(Errors error) { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); // Send two async commits and fail the first one with an error. // This should cause a coordinator disconnect which will cancel the second request. MockCommitCallback firstCommitCallback = new MockCommitCallback(); MockCommitCallback secondCommitCallback = new MockCommitCallback(); coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), firstCommitCallback); coordinator.commitOffsetsAsync(singletonMap(t1p, new OffsetAndMetadata(100L)), secondCommitCallback); respondToOffsetCommitRequest(singletonMap(t1p, 100L), error); consumerClient.pollNoWakeup(); consumerClient.pollNoWakeup(); // second poll since coordinator disconnect is async coordinator.invokeCompletedOffsetCommitCallbacks(); assertTrue(coordinator.coordinatorUnknown()); assertTrue(firstCommitCallback.exception instanceof RetriableCommitFailedException); assertTrue(secondCommitCallback.exception instanceof RetriableCommitFailedException); }
@Test public void testDisconnectWithInFlightRequests() { RequestFuture<ClientResponse> future = consumerClient.send(node, heartbeat()); consumerClient.pollNoWakeup(); assertTrue(consumerClient.hasPendingRequests(node)); assertTrue(client.hasInFlightRequests(node.idString())); consumerClient.disconnectAsync(node); consumerClient.pollNoWakeup(); assertTrue(future.failed()); assertTrue(future.exception() instanceof DisconnectException); }
@Test public void testUpdateFetchPositionResetToLatestOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testIdempotentResetWithInFlightReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // Send the ListOffsets request to reset the position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(client.hasInFlightRequests()); // Now we get a seek from the user subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.respond(listOffsetResponse(Errors.NONE, 1L, 5L)); consumerClient.pollNoWakeup(); assertFalse(client.hasInFlightRequests()); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(5L, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionResetToEarliestOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionResetToDefaultOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionOfPausedPartitionsWithoutAValidPosition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0); subscriptions.pause(tp0); // paused partition does not have a valid position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertFalse(subscriptions.hasValidPosition(tp0)); }
@Test public void testAssignmentChangeWithInFlightReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // Send the ListOffsets request to reset the position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(client.hasInFlightRequests()); // Now we have an assignment change subscriptions.assignFromUser(singleton(tp1)); // The response returns and is discarded client.respond(listOffsetResponse(Errors.NONE, 1L, 5L)); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingResponses()); assertFalse(client.hasInFlightRequests()); assertFalse(subscriptions.isAssigned(tp0)); }
@Test public void testSeekWithInFlightReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // Send the ListOffsets request to reset the position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(client.hasInFlightRequests()); // Now we get a seek from the user subscriptions.seek(tp0, 237); // The response returns and is discarded client.respond(listOffsetResponse(Errors.NONE, 1L, 5L)); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingResponses()); assertFalse(client.hasInFlightRequests()); assertEquals(237L, subscriptions.position(tp0).longValue()); }
@Test public void testResetOffsetsMetadataRefresh() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // First fetch fails with stale metadata client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NOT_LEADER_FOR_PARTITION, 1L, 5L), false); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); // Expect a metadata refresh client.prepareMetadataUpdate(initialUpdateResponse); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingMetadataUpdates()); // Next fetch succeeds time.sleep(retryBackoffMs); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testTimeoutUnsentRequest() { // Delay connection to the node so that the request remains unsent client.delayReady(node, 1000); RequestFuture<ClientResponse> future = consumerClient.send(node, heartbeat(), 500); consumerClient.pollNoWakeup(); // Ensure the request is pending, but hasn't been sent assertTrue(consumerClient.hasPendingRequests()); assertFalse(client.hasInFlightRequests()); time.sleep(501); consumerClient.pollNoWakeup(); assertFalse(consumerClient.hasPendingRequests()); assertTrue(future.failed()); assertTrue(future.exception() instanceof TimeoutException); }
@Test public void testUpdateFetchPositionOfPausedPartitionsRequiringOffsetReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.pause(tp0); // paused partition does not have a valid position subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 10L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertTrue(subscriptions.hasValidPosition(tp0)); assertEquals(10, subscriptions.position(tp0).longValue()); }
@Test public void testChangeResetWithInFlightReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); // Send the ListOffsets request to reset the position fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); assertTrue(client.hasInFlightRequests()); // Now we get a seek from the user subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); // The response returns and is discarded client.respond(listOffsetResponse(Errors.NONE, 1L, 5L)); consumerClient.pollNoWakeup(); assertFalse(client.hasPendingResponses()); assertFalse(client.hasInFlightRequests()); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(OffsetResetStrategy.EARLIEST, subscriptions.resetStrategy(tp0)); }
@Test public void testResetOffsetsSkipsBlackedOutConnections() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); // Check that we skip sending the ListOffset request when the node is blacked out client.blackout(node, 500); fetcher.resetOffsetsIfNeeded(); assertEquals(0, consumerClient.pendingRequestCount()); consumerClient.pollNoWakeup(); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(OffsetResetStrategy.EARLIEST, subscriptions.resetStrategy(tp0)); time.sleep(500); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testListOffsetsSendsIsolationLevel() { for (final IsolationLevel isolationLevel : IsolationLevel.values()) { Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, isolationLevel); subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { ListOffsetRequest request = (ListOffsetRequest) body; return request.isolationLevel() == isolationLevel; } }, listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); } }
@Test public void testDisconnectWithUnsentRequests() { RequestFuture<ClientResponse> future = consumerClient.send(node, heartbeat()); assertTrue(consumerClient.hasPendingRequests(node)); assertFalse(client.hasInFlightRequests(node.idString())); consumerClient.disconnectAsync(node); consumerClient.pollNoWakeup(); assertTrue(future.failed()); assertTrue(future.exception() instanceof DisconnectException); }