int implied = sessionPartitions.size() - response.responseData().size(); if (implied > 0) { return String.format(" with %d response partition(s), %d implied partition(s)", response.responseData().size(), implied); } else { return String.format(" with %d response partition(s)", response.responseData().size()); append(Utils.join(response.responseData().keySet(), ", ")). append(")"); String prefix = ", implied=("; String suffix = ""; for (TopicPartition partition : sessionPartitions.keySet()) { if (!response.responseData().containsKey(partition)) { bld.append(prefix); bld.append(partition);
/** * Verify that the partitions in an incremental fetch response are contained in the session. * * @param response The response. * @return True if the incremental fetch response partitions are valid. */ private String verifyIncrementalFetchResponsePartitions(FetchResponse<?> response) { Set<TopicPartition> extra = findMissing(response.responseData().keySet(), sessionPartitions.keySet()); if (!extra.isEmpty()) { StringBuilder bld = new StringBuilder(); bld.append("extra=(").append(Utils.join(extra, ", ")).append("), "); bld.append("response=(").append( Utils.join(response.responseData().keySet(), ", ")).append("), "); return bld.toString(); } return null; }
/** * Verify that a full fetch response contains all the partitions in the fetch session. * * @param response The response. * @return True if the full fetch response partitions are valid. */ private String verifyFullFetchResponsePartitions(FetchResponse<?> response) { StringBuilder bld = new StringBuilder(); Set<TopicPartition> omitted = findMissing(response.responseData().keySet(), sessionPartitions.keySet()); Set<TopicPartition> extra = findMissing(sessionPartitions.keySet(), response.responseData().keySet()); if (!omitted.isEmpty()) { bld.append("omitted=(").append(Utils.join(omitted, ", ")).append(", "); } if (!extra.isEmpty()) { bld.append("extra=(").append(Utils.join(extra, ", ")).append(", "); } if ((!omitted.isEmpty()) || (!extra.isEmpty())) { bld.append("response=(").append(Utils.join(response.responseData().keySet(), ", ")); return bld.toString(); } return null; }
@Override public void onSuccess(ClientResponse resp) { synchronized (Fetcher.this) { @SuppressWarnings("unchecked") FetchResponse<Records> response = (FetchResponse<Records>) resp.responseBody(); FetchSessionHandler handler = sessionHandler(fetchTarget.id()); if (handler == null) { log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.", fetchTarget.id()); return; } if (!handler.handleResponse(response)) { return; } Set<TopicPartition> partitions = new HashSet<>(response.responseData().keySet()); FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions); for (Map.Entry<TopicPartition, FetchResponse.PartitionData<Records>> entry : response.responseData().entrySet()) { TopicPartition partition = entry.getKey(); long fetchOffset = data.sessionPartitions().get(partition).fetchOffset; FetchResponse.PartitionData<Records> fetchData = entry.getValue(); log.debug("Fetch {} at offset {} for partition {} returned fetch data {}", isolationLevel, fetchOffset, partition, fetchData); completedFetches.add(new CompletedFetch(partition, fetchOffset, fetchData, metricAggregator, resp.requestHeader().apiVersion())); } sensors.fetchLatency.record(resp.requestLatencyMs()); } }
@Test public void testFetchResponseV4() { LinkedHashMap<TopicPartition, FetchResponse.PartitionData<MemoryRecords>> responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10)); List<FetchResponse.AbortedTransaction> abortedTransactions = asList( new FetchResponse.AbortedTransaction(10, 100), new FetchResponse.AbortedTransaction(15, 50) ); responseData.put(new TopicPartition("bar", 0), new FetchResponse.PartitionData<>(Errors.NONE, 100000, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, abortedTransactions, records)); responseData.put(new TopicPartition("bar", 1), new FetchResponse.PartitionData<>(Errors.NONE, 900000, 5, FetchResponse.INVALID_LOG_START_OFFSET, null, records)); responseData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData<>(Errors.NONE, 70000, 6, FetchResponse.INVALID_LOG_START_OFFSET, Collections.emptyList(), records)); FetchResponse<MemoryRecords> response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); FetchResponse deserialized = FetchResponse.parse(toBuffer(response.toStruct((short) 4)), (short) 4); assertEquals(responseData, deserialized.responseData()); }
@Test public void fetchResponseVersionTest() { LinkedHashMap<TopicPartition, FetchResponse.PartitionData<MemoryRecords>> responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10)); responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>( Errors.NONE, 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); FetchResponse<MemoryRecords> v0Response = new FetchResponse<>(Errors.NONE, responseData, 0, INVALID_SESSION_ID); FetchResponse<MemoryRecords> v1Response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs()); assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs()); assertEquals("Should use schema version 0", ApiKeys.FETCH.responseSchema((short) 0), v0Response.toStruct((short) 0).schema()); assertEquals("Should use schema version 1", ApiKeys.FETCH.responseSchema((short) 1), v1Response.toStruct((short) 1).schema()); assertEquals("Response data does not match", responseData, v0Response.responseData()); assertEquals("Response data does not match", responseData, v1Response.responseData()); }