private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException( String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException(String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
if (fetchResponse.hasError()) { String exception = ""; List<KafkaTopicPartitionState<TopicAndPartition>> partitionsToGetOffsetsFor = new ArrayList<>();
.addFetch(_topic, _partition, startOffset, 500000).build()); if (!fetchResponse.hasError()) { final Iterable<MessageAndOffset> messageAndOffsetIterable = buildOffsetFilteringIterable(fetchResponse.messageSet(_topic, _partition), startOffset, endOffset);
if (fetchResponse.hasError()) { short code = fetchResponse.errorCode(topic, partition); if (code == ErrorMapping.OffsetOutOfRangeCode() && config.resetOffsetIfOutOfRange) {
if (fetchResponse.hasError()) { short errorCode = fetchResponse.errorCode(split.getTopicName(), split.getPartitionId()); log.warn("Fetch response has error: %d", errorCode);
.build(); FetchResponse response = consumer.fetch(request); if (response.hasError()) { consumer.close(); int errorCode = response.errorCode(topicPartition.getTopic(), topicPartition.getPartition());
if (response == null || response.hasError()) { short errorCode = response != null ? response.errorCode(topic, partitionId) : ErrorMapping.UnknownCode(); log.warn("fetch %s - %s with offset %s encounters error: [%s]", topic, partitionId, offset, errorCode);
private boolean refreshTopicMetadataAndRetryFetch(FetchRequest fetchRequest, long tempTime) { try { refreshTopicMetadata(); FetchResponse fetchResponse = simpleConsumer.fetch(fetchRequest); if (fetchResponse.hasError()) { log.warn("Error encountered during fetch request retry from Kafka"); log.warn("Error Code generated : " + fetchResponse.errorCode(kafkaRequest.getTopic(), kafkaRequest.getPartition())); return false; } return processFetchResponse(fetchResponse, tempTime); } catch (Exception e) { log.info("Exception generated during fetch for topic " + kafkaRequest.getTopic() + ". This topic will be skipped."); return false; } }
private FetchResponse mockFetchResponse(List<MyMessage> myMessages) { FetchResponse fetchResponse = EasyMock.createMock(FetchResponse.class); EasyMock.expect(fetchResponse.hasError()).andReturn(false).times(1); List<Message> messages = new ArrayList<Message>(); for (MyMessage myMessage:myMessages) { String payload = gson.toJson(myMessage); String msgKey = Integer.toString(PARTITION_1_ID); Message message = new Message(payload.getBytes(), msgKey.getBytes()); messages.add(message); } ByteBufferMessageSet messageSet = new ByteBufferMessageSet(messages); EasyMock.expect(fetchResponse.messageSet(EasyMock.anyString(), EasyMock.anyInt())).andReturn(messageSet).times(1); mocks.add(fetchResponse); return fetchResponse; }
try { fetchResponse = simpleConsumer.fetch(fetchRequest); if (fetchResponse.hasError()) { String message = "Error Code generated : " + fetchResponse.errorCode(kafkaRequest.getTopic(), kafkaRequest.getPartition())
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException( String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException(String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException(String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException( String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private boolean refreshTopicMetadataAndRetryFetch(FetchRequest fetchRequest, long tempTime) { try { refreshTopicMetadata(); FetchResponse fetchResponse = simpleConsumer.fetch(fetchRequest); if (fetchResponse.hasError()) { log.warn("Error encountered during fetch request retry from Kafka"); log.warn("Error Code generated : " + fetchResponse.errorCode(kafkaRequest.getTopic(), kafkaRequest.getPartition())); return false; } return processFetchResponse(fetchResponse, tempTime); } catch (Exception e) { log.info("Exception generated during fetch for topic " + kafkaRequest.getTopic() + ". This topic will be skipped."); return false; } }
private ByteBufferMessageSet fetchMessageSet(long fetchOffset) throws OffsetOutOfRangeException { Preconditions.checkArgument(fetchOffset >= 0, String.format("Illegal fetch offset %d", fetchOffset)); int failureCount = 0; while (true) { SimpleConsumer consumer = getConsumer(); FetchRequest req = new FetchRequestBuilder() .clientId(clientName) .addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES) .maxWait(fetchTimeoutMs) .build(); FetchResponse fetchResponse = consumer.fetch(req); if (!fetchResponse.hasError()) { return fetchResponse.messageSet(topic, partition); } short errorCode = fetchResponse.errorCode(topic, partition); if (++failureCount >= MAX_KAFKA_FETCH_RETRIES) { throw new RuntimeException( String.format("Error fetching data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, errorCode)); } LOG.warn("Error fetching data from broker {}:{} for topic {}, partition {}. Error code: {}", consumer.host(), consumer.port(), topic, partition, errorCode); if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) { throw new OffsetOutOfRangeException(String.format( "Requested offset %d is out of range for topic %s partition %d", fetchOffset, topic, partition)); } closeConsumer(); } }
private ByteBufferMessageSet fetchMessageSet(long fetchOffset) throws OffsetOutOfRangeException { Preconditions.checkArgument(fetchOffset >= 0, String.format("Illegal fetch offset %d", fetchOffset)); int failureCount = 0; while (true) { SimpleConsumer consumer = getConsumer(); FetchRequest req = new FetchRequestBuilder() .clientId(clientName) .addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES) .maxWait(fetchTimeoutMs) .build(); FetchResponse fetchResponse = consumer.fetch(req); if (!fetchResponse.hasError()) { return fetchResponse.messageSet(topic, partition); } short errorCode = fetchResponse.errorCode(topic, partition); if (++failureCount >= MAX_KAFKA_FETCH_RETRIES) { throw new RuntimeException( String.format("Error fetching data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, errorCode)); } LOG.warn("Error fetching data from broker {}:{} for topic {}, partition {}. Error code: {}", consumer.host(), consumer.port(), topic, partition, errorCode); if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) { throw new OffsetOutOfRangeException(String.format( "Requested offset %d is out of range for topic %s partition %d", fetchOffset, topic, partition)); } closeConsumer(); } }
/** * Fetches messages from the given topic, partition and offset using the provided {@link SimpleConsumer}. * * @return A {@link ByteBufferMessageSet} of the given topic, partition for messages fetched * * @throws OffsetOutOfRangeException if the given offset is out of range. * @throws NotLeaderForPartitionException if the broker that the consumer is talking to is not the leader * for the given topic and partition. * @throws UnknownTopicOrPartitionException if the topic or partition is not known by the Kafka server * @throws UnknownServerException if the Kafka server responded with error. */ public static ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, String topic, int partition, int fetchSize, long requestOffset) throws KafkaException { FetchRequest req = new FetchRequestBuilder() .clientId(consumer.clientId()) .addFetch(topic, partition, requestOffset, fetchSize) .build(); FetchResponse fetchResponse = consumer.fetch(req); if (fetchResponse.hasError()) { throw Errors.forCode(fetchResponse.errorCode(topic, partition)).exception(); } return fetchResponse.messageSet(topic, partition); }
/** * Fetches messages from the given topic, partition and offset using the provided {@link SimpleConsumer}. * * @return A {@link ByteBufferMessageSet} of the given topic, partition for messages fetched * * @throws OffsetOutOfRangeException if the given offset is out of range. * @throws NotLeaderForPartitionException if the broker that the consumer is talking to is not the leader * for the given topic and partition. * @throws UnknownTopicOrPartitionException if the topic or partition is not known by the Kafka server * @throws UnknownServerException if the Kafka server responded with error. */ public static ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, String topic, int partition, int fetchSize, long requestOffset) throws KafkaException { FetchRequest req = new FetchRequestBuilder() .clientId(consumer.clientId()) .addFetch(topic, partition, requestOffset, fetchSize) .build(); FetchResponse fetchResponse = consumer.fetch(req); if (fetchResponse.hasError()) { throw Errors.forCode(fetchResponse.errorCode(topic, partition)).exception(); } return fetchResponse.messageSet(topic, partition); }