private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId) { TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId); // The API implies that this will always return all of the offsets. So it seems a partition can not have // more than Integer.MAX_VALUE-1 segments. // // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value // should not be 0. PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), Integer.MAX_VALUE); OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); if (offsetResponse.hasError()) { short errorCode = offsetResponse.errorCode(topicName, partitionId); throw new RuntimeException("could not fetch data from Kafka, error code is '" + errorCode + "'"); } return offsetResponse.offsets(topicName, partitionId); }
@Override public long getLastOffset(long time) { SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000, 1024 * 1024, "hadoop-etl"); Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(time, 1)); OffsetResponse response = consumer .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), "hadoop-etl")); long[] endOffset = response.offsets(topic, partition); consumer.close(); if (endOffset.length == 0) { log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition); } this.latestOffset = endOffset[0]; return endOffset[0]; }
private static long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition, response.errorCode(topicAndPartition.topic(), topicAndPartition.partition())); return -1; } long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition()); return offsets[0]; } }
OffsetResponse response; while (true) { kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( partitionToRequestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); response = consumer.getOffsetsBefore(request); if (response.hasError()) { StringBuilder exception = new StringBuilder(); for (KafkaTopicPartitionState<TopicAndPartition> part : partitionStates) { short code; if ((code = response.errorCode(part.getTopic(), part.getPartition())) != ErrorMapping.NoError()) { exception.append("\nException for topic=").append(part.getTopic()) .append(" partition=").append(part.getPartition()).append(": ") final long offset = response.offsets(part.getTopic(), part.getPartition())[0];
private long getOffsets(Node leader, String topic, int partitionId, long time) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId); SimpleConsumer consumer = new SimpleConsumer(leader.host(), leader.port(), 10000, 1024, "Kafka-zk-simpleconsumer"); PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(time, 10000); OffsetRequest offsetRequest = new OffsetRequest( ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); if (offsetResponse.hasError()) { short errorCode = offsetResponse.errorCode(topic, partitionId); log.warn(format("Offset response has error: %d", errorCode)); throw new ApiException( "could not fetch data from Kafka, error code is '" + errorCode + "'Exception Message:" + offsetResponse.toString()); } long[] offsets = offsetResponse.offsets(topic, partitionId); consumer.close(); return offsets[0]; }
public long getOffset(String topic, int partition, long startOffsetTime) { SimpleConsumer simpleConsumer = findLeaderConsumer(partition); if (simpleConsumer == null) { LOG.error("Error consumer is null get offset from partition:" + partition); return -1; } TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId()); long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition); if (offsets.length > 0) { return offsets[0]; } else { return NO_OFFSET; } }
"hive_kafka_client"); PartitionOffsetRequestInfo partitionLatestOffsetRequestInfo = new PartitionOffsetRequestInfo( kafka.api.OffsetRequest.LatestTime(), 1); PartitionOffsetRequestInfo partitionEarliestOffsetRequestInfo = new PartitionOffsetRequestInfo( kafka.api.OffsetRequest.EarliestTime(), 1); Map<TopicAndPartition, PartitionOffsetRequestInfo> latestOffsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); Map<TopicAndPartition, PartitionOffsetRequestInfo> earliestOffsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); .getOffsetsBefore(new OffsetRequest(latestOffsetInfo, kafka.api.OffsetRequest.CurrentVersion(), "hive_kafka_client")); OffsetResponse earliestOffsetResponse = consumer .getOffsetsBefore(new OffsetRequest(earliestOffsetInfo, kafka.api.OffsetRequest.CurrentVersion(), "hive_kafka_client")); consumer.close(); for (TopicAndPartition topicAndPartition : topicAndPartitions) { long latestOffset = latestOffsetResponse.offsets( topicAndPartition.topic(), topicAndPartition.partition())[0]; long earliestOffset = earliestOffsetResponse.offsets( topicAndPartition.topic(), topicAndPartition.partition())[0];
Arrays.toString(topics), endpoint.host()), err); } finally { consumer.close(); requestInfo = brokerRequests.get(leader); requestInfo.put(new TopicAndPartition(metadata.topic(), partition.partitionId()), new PartitionOffsetRequestInfo( time, 1)); OffsetRequest offsetRequest = new OffsetRequest(brokerRequest.getValue(), kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID); offsetResponse = simpleConsumer.getOffsetsBefore(offsetRequest); } finally { simpleConsumer.close(); TopicPartition topicPartition = new TopicPartition(topicAndPartition.topic(), topicAndPartition.partition()); long[] offsets = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition()); long offset; if (time == kafka.api.OffsetRequest.EarliestTime()) throw new IllegalStateException("We requested the earliest offsets for topic [" + topicAndPartition.topic() + "] but Kafka returned no values"); kafka.api.OffsetRequest.EarliestTime(), topicAndPartition.topic());
private long getEarliestOffset() { // return kafka.api.OffsetRequest.EarliestTime(); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>(); requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(EARLIEST_TIME, 1)); OffsetRequest offsetRequest = new OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID); if (earliestOffset <= 0) { OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); earliestOffset = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0]; } return earliestOffset; }
protected OffsetResponse getLatestOffsetResponse(SimpleConsumer consumer, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo, JobContext context) { for (int i = 0; i < NUM_TRIES_FETCH_FROM_LEADER; i++) { try { OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), CamusJob .getKafkaClientName(context))); if (offsetResponse.hasError()) { throw new RuntimeException("offsetReponse has error."); } return offsetResponse; } catch (Exception e) { log.warn("Fetching offset from leader " + consumer.host() + ":" + consumer.port() + " has failed " + (i + 1) + " time(s). Reason: " + e.getMessage() + " " + (NUM_TRIES_FETCH_FROM_LEADER - i - 1) + " retries left."); if (i < NUM_TRIES_FETCH_FROM_LEADER - 1) { try { Thread.sleep((long) (Math.random() * (i + 1) * 1000)); } catch (InterruptedException e1) { log.error("Caught interrupted exception between retries of getting latest offsets. " + e1.getMessage()); } } } } return null; }
private OffsetResponse sendOffsetRequest(Integer brokerId, TopicVO topic, PartitionOffsetRequestInfo requestInfo, List<TopicPartitionVO> brokerPartitions) { final OffsetRequest offsetRequest = new OffsetRequest( brokerPartitions.stream() .collect(Collectors.toMap( partition -> new TopicAndPartition(topic.getName(), partition.getId()), partition -> requestInfo)), (short) 0, clientId()); LOG.debug("Sending offset request: {}", offsetRequest); return retryTemplate.execute( context -> brokerChannel(brokerId) .execute(channel -> { channel.send(offsetRequest.underlying()); final kafka.api.OffsetResponse underlyingResponse = kafka.api.OffsetResponse.readFrom(channel.receive().buffer()); LOG.debug("Received offset response: {}", underlyingResponse); return new OffsetResponse(underlyingResponse); })); }
@Override public long getLastOffset(long time) { SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000, 1024 * 1024, "hadoop-etl"); Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(time, 1)); OffsetResponse response = consumer .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), "hadoop-etl")); long[] endOffset = response.offsets(topic, partition); consumer.close(); if (endOffset.length == 0) { log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition); } this.latestOffset = endOffset[0]; return endOffset[0]; }
private long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition, response.errorCode(topicAndPartition.topic(), topicAndPartition.partition())); return -1; } long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition()); return offsets[0]; }
private long getOffset(KafkaPartition partition, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo) throws KafkaOffsetRetrievalFailureException { SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort()); for (int i = 0; i < this.fetchOffsetRetries; i++) { try { OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo, kafka.api.OffsetRequest.CurrentVersion(), this.clientName)); if (offsetResponse.hasError()) { throw new RuntimeException("offsetReponse has error: " + offsetResponse.errorCode(partition.getTopicName(), partition.getId())); } return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0]; } catch (Exception e) { log.warn(String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1, e)); if (i < this.fetchOffsetRetries - 1) { try { Thread.sleep((long) ((i + Math.random()) * 1000)); } catch (InterruptedException e2) { log.error("Caught interrupted exception between retries of getting latest offsets. " + e2); } } } } throw new KafkaOffsetRetrievalFailureException(String.format("Fetching offset for partition %s has failed.", partition)); }
public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); OffsetRequest request = new OffsetRequest( requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition); if (offsets.length > 0) { return offsets[0]; } else { return NO_OFFSET; } }
private long getLatestOffset() { // return kafka.api.OffsetRequest.LatestTime(); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>(); requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(LATEST_TIME, 1)); OffsetRequest offsetRequest = new OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID); if (latestOffset <= 0) { OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); latestOffset = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0]; } return latestOffset; }
protected OffsetResponse getLatestOffsetResponse(SimpleConsumer consumer, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo, JobContext context) { for (int i = 0; i < NUM_TRIES_FETCH_FROM_LEADER; i++) { try { OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), CamusJob .getKafkaClientName(context))); if (offsetResponse.hasError()) { throw new RuntimeException("offsetReponse has error."); } return offsetResponse; } catch (Exception e) { log.warn("Fetching offset from leader " + consumer.host() + ":" + consumer.port() + " has failed " + (i + 1) + " time(s). Reason: " + e.getMessage() + " " + (NUM_TRIES_FETCH_FROM_LEADER - i - 1) + " retries left."); if (i < NUM_TRIES_FETCH_FROM_LEADER - 1) { try { Thread.sleep((long) (Math.random() * (i + 1) * 1000)); } catch (InterruptedException e1) { log.error("Caught interrupted exception between retries of getting latest offsets. " + e1.getMessage()); } } } } return null; }
private long getOffset(boolean earliest) throws InterruptedException { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put( topicAndPartition, new PartitionOffsetRequestInfo( earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1 ) ); OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId); OffsetResponse response; try { response = consumer.getOffsetsBefore(request); } catch (Exception e) { ensureNotInterrupted(e); log.error(e, "caught exception in getOffsetsBefore [%s] - [%s]", topic, partitionId); return -1; } if (response.hasError()) { log.error( "error fetching data Offset from the Broker [%s]. reason: [%s]", leaderBroker.host(), response.errorCode(topic, partitionId) ); return -1; } long[] offsets = response.offsets(topic, partitionId); return earliest ? offsets[0] : offsets[offsets.length - 1]; }
@Override public long getEarliestOffset() { if (this.earliestOffset == -2 && uri != null) { // TODO : Make the hardcoded paramters configurable SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000, 1024 * 1024, "hadoop-etl"); Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); OffsetResponse response = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), "hadoop-etl")); long[] endOffset = response.offsets(topic, partition); consumer.close(); this.earliestOffset = endOffset[0]; return endOffset[0]; } else { return this.earliestOffset; } }
public long getLogEndOffset(String topic, int partition) { try { SimpleConsumer consumer = findLeaderConsumer(topic, partition); TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> infoMap = new HashMap<>(); infoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); OffsetRequest request = new OffsetRequest(infoMap, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse response = consumer.getOffsetsBefore(request); // Retrieve offsets from response long[] offsets = response.hasError() ? null : response.offsets(topicAndPartition.topic(), topicAndPartition.partition()); if (offsets == null || offsets.length <= 0) { short errorCode = response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()); // If the topic partition doesn't exists, use offset 0 without logging error. if (errorCode != ErrorMapping.UnknownTopicOrPartitionCode()) { log.warn("Failed to fetch latest offset for {}. Error: {}. Default offset to 0.", topicAndPartition, errorCode); } return 0L; } consumer.close(); return offsets[0]; } catch (Exception ex) { log.error("unable to retrieve offset {}", new Object[]{ex}); return -1; } }