frb.maxWait(maxWait); frb.minBytes(minBytes);
new FetchRequestBuilder().minBytes(100000).maxWait(timeoutMillis) .addFetch(_topic, _partition, startOffset, 500000).build());
.maxWait(config.fetchWaitMaxMs).build(); FetchResponse fetchResponse = null; SimpleConsumer simpleConsumer = null;
.clientId(clientId) .addFetch(topic, partitionId, offset, FETCH_SIZE) .maxWait(timeoutMs) .minBytes(1) .build();
.maxWait(5000) // todo: make configurable .minBytes(1);
private ByteBufferMessageSet fetchMessageSet(long fetchOffset) throws OffsetOutOfRangeException { Preconditions.checkArgument(fetchOffset >= 0, String.format("Illegal fetch offset %d", fetchOffset)); int failureCount = 0; while (true) { SimpleConsumer consumer = getConsumer(); FetchRequest req = new FetchRequestBuilder() .clientId(clientName) .addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES) .maxWait(fetchTimeoutMs) .build(); FetchResponse fetchResponse = consumer.fetch(req); if (!fetchResponse.hasError()) { return fetchResponse.messageSet(topic, partition); } short errorCode = fetchResponse.errorCode(topic, partition); if (++failureCount >= MAX_KAFKA_FETCH_RETRIES) { throw new RuntimeException( String.format("Error fetching data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, errorCode)); } LOG.warn("Error fetching data from broker {}:{} for topic {}, partition {}. Error code: {}", consumer.host(), consumer.port(), topic, partition, errorCode); if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) { throw new OffsetOutOfRangeException(String.format( "Requested offset %d is out of range for topic %s partition %d", fetchOffset, topic, partition)); } closeConsumer(); } }
private ByteBufferMessageSet fetchMessageSet(long fetchOffset) throws OffsetOutOfRangeException { Preconditions.checkArgument(fetchOffset >= 0, String.format("Illegal fetch offset %d", fetchOffset)); int failureCount = 0; while (true) { SimpleConsumer consumer = getConsumer(); FetchRequest req = new FetchRequestBuilder() .clientId(clientName) .addFetch(topic, partition, fetchOffset, BUFFER_SIZE_BYTES) .maxWait(fetchTimeoutMs) .build(); FetchResponse fetchResponse = consumer.fetch(req); if (!fetchResponse.hasError()) { return fetchResponse.messageSet(topic, partition); } short errorCode = fetchResponse.errorCode(topic, partition); if (++failureCount >= MAX_KAFKA_FETCH_RETRIES) { throw new RuntimeException( String.format("Error fetching data from broker %s:%d for topic %s, partition %d. Error code: %d", consumer.host(), consumer.port(), topic, partition, errorCode)); } LOG.warn("Error fetching data from broker {}:{} for topic {}, partition {}. Error code: {}", consumer.host(), consumer.port(), topic, partition, errorCode); if (errorCode == ErrorMapping.OffsetOutOfRangeCode()) { throw new OffsetOutOfRangeException(String.format( "Requested offset %d is out of range for topic %s partition %d", fetchOffset, topic, partition)); } closeConsumer(); } }
.clientId(FETCH_CLIENT_NAME) .maxWait(FETCH_MAX_WAIT_MS) .minBytes(ConsumerConfig.MinFetchBytes()) .addFetch(topic, partition, readOffset, FETCH_BUFFER_SIZE)
.clientId(FETCH_CLIENT_NAME) .maxWait(FETCH_MAX_WAIT_MS) .minBytes(ConsumerConfig.MinFetchBytes()) .addFetch(topic, partition, readOffset, FETCH_BUFFER_SIZE)
.addFetch(topic, partition, startOffset, bufferSizeInUse).maxWait(DEFAULT_TIME_TO_WAIT) .minBytes(DEFAULT_MIN_BYTES_TO_WAIT).build(); FetchResponse fetchResponse = consumer.fetch(fetchRequest);
/** * Makes a call to kafka to fetch messages. */ private FetchResponse fetchMessages(SimpleConsumer consumer, long offset) { FetchRequest request = new FetchRequestBuilder() .clientId(consumer.clientId()) .addFetch(topicPart.getTopic(), topicPart.getPartition(), offset, FETCH_SIZE) .maxWait(MAX_WAIT) .build(); return consumer.fetch(request); }
FetchRequestBuilder builder = new FetchRequestBuilder(); FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes). clientId(config.clientId).maxWait(config.fetchMaxWait).minBytes(config.minFetchByte).build(); FetchResponse fetchResponse; try {
/** * Makes a call to kafka to fetch messages. */ private FetchResponse fetchMessages(SimpleConsumer consumer, long offset) { FetchRequest request = new FetchRequestBuilder() .clientId(consumer.clientId()) .addFetch(topicPart.getTopic(), topicPart.getPartition(), offset, FETCH_SIZE) .maxWait(MAX_WAIT) .build(); return consumer.fetch(request); }
frb.maxWait(maxWait); frb.minBytes(minBytes);
frb.maxWait(maxWait); frb.minBytes(minBytes);
frb.maxWait(maxWait); frb.minBytes(minBytes);
.maxWait(config.fetchWaitMaxMs).build(); FetchResponse fetchResponse = null; SimpleConsumer simpleConsumer = null;