kafka.javaapi.OffsetRequest.<init> java code examples

Common ways to obtain OffsetRequest

private void myMethod () {
     OffsetRequest o =Map requestInfo;String clientName;new kafka.javaapi.OffsetRequest(requestInfo, OffsetRequest.CurrentVersion(), clientName)
Map map;SimpleConsumer simpleConsumer;new kafka.javaapi.OffsetRequest(map, OffsetRequest.CurrentVersion(), simpleConsumer.clientId())
Smart code suggestions by Codota
}

public long getOffset(String topic, int partition, long startOffsetTime) {
  SimpleConsumer simpleConsumer = findLeaderConsumer(partition);
  if (simpleConsumer == null) {
    LOG.error("Error consumer is null get offset from partition:" + partition);
    return -1;
  }
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
  OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());
  long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
  if (offsets.length > 0) {
    return offsets[0];
  } else {
    return NO_OFFSET;
  }
}

private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId)
{
  TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId);
  // The API implies that this will always return all of the offsets. So it seems a partition can not have
  // more than Integer.MAX_VALUE-1 segments.
  //
  // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value
  // should not be 0.
  PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), Integer.MAX_VALUE);
  OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);
  if (offsetResponse.hasError()) {
    short errorCode = offsetResponse.errorCode(topicName, partitionId);
    throw new RuntimeException("could not fetch data from Kafka, error code is '" + errorCode + "'");
  }
  return offsetResponse.offsets(topicName, partitionId);
}

OffsetResponse response;
while (true) {
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
    partitionToRequestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  response = consumer.getOffsetsBefore(request);

private long getOffset(KafkaPartition partition, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
  throws KafkaOffsetRetrievalFailureException {
 SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
 for (int i = 0; i < this.fetchOffsetRetries; i++) {
  try {
   OffsetResponse offsetResponse =
     consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo, kafka.api.OffsetRequest.CurrentVersion(),
       this.clientName));
   if (offsetResponse.hasError()) {
    throw new RuntimeException("offsetReponse has error: "
      + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
   }
   return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
  } catch (Exception e) {
   log.warn(String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1,
     e));
   if (i < this.fetchOffsetRetries - 1) {
    try {
     Thread.sleep((long) ((i + Math.random()) * 1000));
    } catch (InterruptedException e2) {
     log.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
    }
   }
  }
 }
 throw new KafkaOffsetRetrievalFailureException(String.format("Fetching offset for partition %s has failed.",
   partition));
}

private long getOffset(KafkaPartition partition,
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
  throws KafkaOffsetRetrievalFailureException {
 SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
 for (int i = 0; i < this.fetchOffsetRetries; i++) {
  try {
   OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo,
     kafka.api.OffsetRequest.CurrentVersion(), this.clientName));
   if (offsetResponse.hasError()) {
    throw new RuntimeException(
      "offsetReponse has error: " + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
   }
   return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
  } catch (Exception e) {
   LOG.warn(
     String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1, e));
   if (i < this.fetchOffsetRetries - 1) {
    try {
     Thread.sleep((long) ((i + Math.random()) * 1000));
    } catch (InterruptedException e2) {
     LOG.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
    }
   }
  }
 }
 throw new KafkaOffsetRetrievalFailureException(
   String.format("Fetching offset for partition %s has failed.", partition));
}

OffsetRequest request = new OffsetRequest(Collections.singletonMap(new TopicAndPartition(_topic, _partition),
  new PartitionOffsetRequestInfo(offsetRequestTime, 1)), kafka.api.OffsetRequest.CurrentVersion(), _clientId);
OffsetResponse offsetResponse;

private long getOffset(boolean earliest) throws InterruptedException
{
 TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
 Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
 requestInfo.put(
   topicAndPartition,
   new PartitionOffsetRequestInfo(
     earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1
   )
 );
 OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);
 OffsetResponse response;
 try {
  response = consumer.getOffsetsBefore(request);
 }
 catch (Exception e) {
  ensureNotInterrupted(e);
  log.error(e, "caught exception in getOffsetsBefore [%s] - [%s]", topic, partitionId);
  return -1;
 }
 if (response.hasError()) {
  log.error(
    "error fetching data Offset from the Broker [%s]. reason: [%s]", leaderBroker.host(),
    response.errorCode(topic, partitionId)
  );
  return -1;
 }
 long[] offsets = response.offsets(topic, partitionId);
 return earliest ? offsets[0] : offsets[offsets.length - 1];
}

private long findLastOffset(TopicPartition topicPartition, SimpleConsumer consumer) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topicPartition.getTopic(),
      topicPartition.getPartition());
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
      new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
      kafka.api.OffsetRequest.LatestTime(), 1));
  final String clientName = getClientName(topicPartition);
  OffsetRequest request = new OffsetRequest(requestInfo,
                       kafka.api.OffsetRequest.CurrentVersion(),
                       clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);
  if (response.hasError()) {
    throw new RuntimeException("Error fetching offset data. Reason: " +
        response.errorCode(topicPartition.getTopic(), topicPartition.getPartition()));
  }
  long[] offsets = response.offsets(topicPartition.getTopic(),
      topicPartition.getPartition());
  return offsets[0] - 1;
}

protected OffsetResponse getLatestOffsetResponse(SimpleConsumer consumer,
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo, JobContext context) {
 for (int i = 0; i < NUM_TRIES_FETCH_FROM_LEADER; i++) {
  try {
   OffsetResponse offsetResponse =
     consumer.getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), CamusJob
       .getKafkaClientName(context)));
   if (offsetResponse.hasError()) {
    throw new RuntimeException("offsetReponse has error.");
   }
   return offsetResponse;
  } catch (Exception e) {
   log.warn("Fetching offset from leader " + consumer.host() + ":" + consumer.port() + " has failed " + (i + 1)
     + " time(s). Reason: " + e.getMessage() + " " + (NUM_TRIES_FETCH_FROM_LEADER - i - 1) + " retries left.");
   if (i < NUM_TRIES_FETCH_FROM_LEADER - 1) {
    try {
     Thread.sleep((long) (Math.random() * (i + 1) * 1000));
    } catch (InterruptedException e1) {
     log.error("Caught interrupted exception between retries of getting latest offsets. " + e1.getMessage());
    }
   }
  }
 }
 return null;
}

 private static long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request =
    new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
   logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition,
     response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()));
   return -1;
  }

  long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition());
  return offsets[0];
 }
}

private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId);
  // The API implies that this will always return all of the offsets. So it seems a partition can not have
  // more than Integer.MAX_VALUE-1 segments.
  //
  // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value
  // should not be 0.
  PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 10000);
  OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);
  if (offsetResponse.hasError()) {
    short errorCode = offsetResponse.errorCode(topicName, partitionId);
    LOGGER.warn(format("Offset response has error: %d", errorCode));
    throw new RakamException("could not fetch data from Kafka, error code is '" + errorCode + "'", HttpResponseStatus.INTERNAL_SERVER_ERROR);
  }
  long[] offsets = offsetResponse.offsets(topicName, partitionId);
  return offsets;
}

public static long getMaxOffset(SimpleConsumer consumer, String topic, int partition) throws Exception {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>();
  requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), "find offset " + System.currentTimeMillis());
  OffsetResponse response = consumer.getOffsetsBefore(request);
  if(response.hasError()) {
    throw new Exception("error fetching data offset , reason = " + response.errorCode(topic, partition));
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}

public static long getMinOffset(SimpleConsumer consumer, String topic, int partition) throws Exception {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>();
  requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), "find offset " + System.currentTimeMillis());
  OffsetResponse response = consumer.getOffsetsBefore(request);
  if(response.hasError()) {
    throw new Exception("error fetching data offset , reason = " + response.errorCode(topic, partition));
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}

private long getEarliestOffset() {
  // return kafka.api.OffsetRequest.EarliestTime();
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>();
  requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(EARLIEST_TIME, 1));
  OffsetRequest offsetRequest = new OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
  if (earliestOffset <= 0) {
    OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);
    earliestOffset = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0];
  }
  return earliestOffset;
}

public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
  OffsetRequest request = new OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition);
  if (offsets.length > 0) {
    return offsets[0];
  } else {
    return NO_OFFSET;
  }
}

private long getLatestOffset() {
  // return kafka.api.OffsetRequest.LatestTime();
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>();
  requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(LATEST_TIME, 1));
  OffsetRequest offsetRequest = new OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
  if (latestOffset <= 0) {
    OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);
    latestOffset = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0];
  }
  return latestOffset;
}

public long getLatestOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, kafka.api.PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);
  if (response.hasError()) {
    throw new RuntimeException("Error fetching data offset from the broker. Reason: " + response.errorCode(topic, partition));
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}

public long getLatestOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, kafka.api.PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);
  if (response.hasError()) {
    throw new RuntimeException("Error fetching data offset from the broker. Reason: " + response.errorCode(topic, partition) );
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}

public long getLatestOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, kafka.api.PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);
  if (response.hasError()) {
    throw new RuntimeException("Error fetching data offset from the broker. Reason: " + response.errorCode(topic, partition) );
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}

private long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
  OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);
  if (response.hasError()) {
    System.out.println("error fetching data offset data the broker. reason: " + response.errorCode(topic, partition));
    return 0;
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}

Popular methods of OffsetRequest

underlying

Popular in Java

Updating database using SQL prepared statement
startActivity (Activity)
orElseThrow (Optional)
runOnUiThread (Activity)
Font (java.awt)
The Font class represents fonts, which are used to render text in a visible way. A font provides the
ConnectException (java.net)
A ConnectException is thrown if a connection cannot be established to a remote host on a specific po
Queue (java.util)
A collection designed for holding elements prior to processing. Besides basic java.util.Collection o
Scanner (java.util)
A parser that parses a text string of primitive types and strings with the help of regular expressio
BlockingQueue (java.util.concurrent)
A java.util.Queue that additionally supports operations that wait for the queue to become non-empty
CountDownLatch (java.util.concurrent)
A synchronization aid that allows one or more threads to wait until a set of operations being perfor

Best Java code snippets using kafka.javaapi.OffsetRequest.<init> (Showing top 20 results out of 369)