public static long getOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) throws IOException { String errMsg = null; Exception lastEx = null; for (int i = 0; i < FETCH_RETRIES; i++) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); try { long offset = consumer.earliestOrLatestOffset(topicAndPartition, whichTime, Request.DebuggingConsumerId()); return offset; } catch (RuntimeException e) { lastEx = e; errMsg = "Failed to getting offset for topic: " + topic + " partition: " + partition + " host: " + consumer.host(); LOG.warn(errMsg, e); try { Thread.sleep((long) (Math.random() * 3000)); } catch (InterruptedException ex) { LOG.warn("Unexpected interruption", ex); } } } throw new IOException(errMsg, lastEx); }
private long getTopicLogSize(String topic, int pid) { Option<Object> o = ZkUtils.getLeaderForPartition(zkClient, topic, pid); if (o.isEmpty() || o.get() == null) { log.error("No broker for partition %s - %s", topic, pid); return 0; } Integer leaderId = Int.unbox(o.get()); SimpleConsumer consumer = consumerMap.get(leaderId); if (consumer == null) { consumer = createSimpleConsumer(leaderId); } // createSimpleConsumer may fail. if (consumer == null) { return 0; } consumerMap.put(leaderId, consumer); TopicAndPartition topicAndPartition = new TopicAndPartition(topic, pid); PartitionOffsetRequestInfo requestInfo = new PartitionOffsetRequestInfo(OffsetRequest.LatestTime(), 1); OffsetRequest request = new OffsetRequest( new Map1<TopicAndPartition, PartitionOffsetRequestInfo>(topicAndPartition, requestInfo), 0, Request.OrdinaryConsumerId() ); OffsetResponse response = consumer.getOffsetsBefore(request); PartitionOffsetsResponse offsetsResponse = response.partitionErrorAndOffsets().get(topicAndPartition).get(); return scala.Long.unbox(offsetsResponse.offsets().head()); }
public static long getOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) throws IOException { String errMsg = null; Exception lastEx = null; for (int i = 0; i < FETCH_RETRIES; i++) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); try { long offset = consumer.earliestOrLatestOffset(topicAndPartition, whichTime, Request.DebuggingConsumerId()); return offset; } catch (RuntimeException e) { lastEx = e; errMsg = "Failed to getting offset for topic: " + topic + " partition: " + partition + " host: " + consumer.host(); LOG.warn(errMsg, e); try { Thread.sleep((long) (Math.random() * 3000)); } catch (InterruptedException ex) { LOG.warn("Unexpected interruption", ex); } } } throw new IOException(errMsg, lastEx); }
FetchRequest req = new FetchRequestBuilder() .clientId(FETCH_CLIENT_NAME) .replicaId(Request.DebuggingConsumerId()) // this consumerId enable reads from follower .maxWait(FETCH_MAX_WAIT_MS) .minBytes(ConsumerConfig.MinFetchBytes())
FetchRequest req = new FetchRequestBuilder() .clientId(FETCH_CLIENT_NAME) .replicaId(Request.DebuggingConsumerId()) // this consumerId enable reads from follower .maxWait(FETCH_MAX_WAIT_MS) .minBytes(ConsumerConfig.MinFetchBytes())