/** * Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. * * This is a blocking call. The consumer does not have to be assigned the partitions. * If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null * will be returned for that partition. * * @param timestampsToSearch the mapping from partition to the timestamp to look up. * * @return a mapping from partition to the timestamp and offset of the first message with timestamp greater * than or equal to the target timestamp. {@code null} will be returned for the partition if there is no * such message. * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic(s). See the exception for more details * @throws IllegalArgumentException if the target timestamp is negative * @throws org.apache.kafka.common.errors.TimeoutException if the offset metadata could not be fetched before * the amount of time allocated by {@code default.api.timeout.ms} expires. * @throws org.apache.kafka.common.errors.UnsupportedVersionException if the broker does not support looking up * the offsets by timestamp */ @Override public Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(Map<TopicPartition, Long> timestampsToSearch) { return offsetsForTimes(timestampsToSearch, Duration.ofMillis(defaultApiTimeoutMs)); }
@Override protected Map<KafkaTopicPartition, Long> fetchOffsetsWithTimestamp( Collection<KafkaTopicPartition> partitions, long timestamp) { Map<TopicPartition, Long> partitionOffsetsRequest = new HashMap<>(partitions.size()); for (KafkaTopicPartition partition : partitions) { partitionOffsetsRequest.put( new TopicPartition(partition.getTopic(), partition.getPartition()), timestamp); } final Map<KafkaTopicPartition, Long> result = new HashMap<>(partitions.size()); // use a short-lived consumer to fetch the offsets; // this is ok because this is a one-time operation that happens only on startup try (KafkaConsumer<?, ?> consumer = new KafkaConsumer(properties)) { for (Map.Entry<TopicPartition, OffsetAndTimestamp> partitionToOffset : consumer.offsetsForTimes(partitionOffsetsRequest).entrySet()) { result.put( new KafkaTopicPartition(partitionToOffset.getKey().topic(), partitionToOffset.getKey().partition()), (partitionToOffset.getValue() == null) ? null : partitionToOffset.getValue().offset()); } } return result; } }
@Override protected Map<KafkaTopicPartition, Long> fetchOffsetsWithTimestamp( Collection<KafkaTopicPartition> partitions, long timestamp) { Map<TopicPartition, Long> partitionOffsetsRequest = new HashMap<>(partitions.size()); for (KafkaTopicPartition partition : partitions) { partitionOffsetsRequest.put( new TopicPartition(partition.getTopic(), partition.getPartition()), timestamp); } final Map<KafkaTopicPartition, Long> result = new HashMap<>(partitions.size()); // use a short-lived consumer to fetch the offsets; // this is ok because this is a one-time operation that happens only on startup try (KafkaConsumer<?, ?> consumer = new KafkaConsumer(properties)) { for (Map.Entry<TopicPartition, OffsetAndTimestamp> partitionToOffset : consumer.offsetsForTimes(partitionOffsetsRequest).entrySet()) { result.put( new KafkaTopicPartition(partitionToOffset.getKey().topic(), partitionToOffset.getKey().partition()), (partitionToOffset.getValue() == null) ? null : partitionToOffset.getValue().offset()); } } return result; }
@Test(expected = AuthenticationException.class) public void testOffsetsForTimesAuthenticationFailure() { final KafkaConsumer<String, String> consumer = consumerWithPendingAuthentication(); consumer.offsetsForTimes(singletonMap(tp0, 0L)); }
Map<TopicPartition, OffsetAndTimestamp> beginningOffsetAndTimestamp = _consumer.offsetsForTimes(beginningTimestamp); for (Map.Entry<TopicPartition, OffsetAndTimestamp> entry: beginningOffsetAndTimestamp.entrySet()) { if (entry.getValue() == null) {
Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestamp = consumer.offsetsForTimes(timePartitionsMap); return Maps.toMap(fullHouse.keySet(), tp -> { KafkaInputSplit existing = fullHouse.get(tp);
private List<KafkaPartitionScanSpec> createScanSpecForTimestamp(String functionName, Long fieldValue) { List<KafkaPartitionScanSpec> scanSpec = Lists.newArrayList(); Map<TopicPartition, Long> timesValMap = Maps.newHashMap(); ImmutableSet<TopicPartition> topicPartitions = fullScanSpec.keySet(); for(TopicPartition partitions : topicPartitions) { timesValMap.put(partitions, functionName.equals("greater_than") ? fieldValue+1 : fieldValue); } Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestamp = kafkaConsumer.offsetsForTimes(timesValMap); for(TopicPartition tp : topicPartitions) { OffsetAndTimestamp value = offsetAndTimestamp.get(tp); //OffsetAndTimestamp is null if there is no offset greater or equal to requested timestamp if(value == null) { scanSpec.add( new KafkaPartitionScanSpec(tp.topic(), tp.partition(), fullScanSpec.get(tp).getEndOffset(), fullScanSpec.get(tp).getEndOffset())); } else { scanSpec.add( new KafkaPartitionScanSpec(tp.topic(), tp.partition(), value.offset(), fullScanSpec.get(tp).getEndOffset())); } } return scanSpec; }
public static Map<TopicPartition, Long> getProcessingStartOffsets(KafkaConsumer<?, ?> kafkaConsumer, String brokerStatsTopic, long startTimestampInMillis) { List<TopicPartition> tpList = kafkaConsumer.partitionsFor(brokerStatsTopic).stream() .map(p->new TopicPartition(p.topic(), p.partition())).collect(Collectors.toList()); Map<TopicPartition, Long> partitionMap = new HashMap<>(); for (TopicPartition topicPartition : tpList) { partitionMap.put(topicPartition, startTimestampInMillis); } Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes = kafkaConsumer .offsetsForTimes(partitionMap); for (Entry<TopicPartition, OffsetAndTimestamp> entry : offsetsForTimes.entrySet()) { partitionMap.put(entry.getKey(), entry.getValue().offset()); } return partitionMap; }
/** * Seek consumer to specific timestamp * @param timestamp Unix timestamp in milliseconds to seek to. */ private void seekToTimestamp(final long timestamp) { // Find offsets for timestamp final Map<TopicPartition, Long> timestampMap = new HashMap<>(); for (final TopicPartition topicPartition: getAllPartitions()) { timestampMap.put(topicPartition, timestamp); } final Map<TopicPartition, OffsetAndTimestamp> offsetMap = kafkaConsumer.offsetsForTimes(timestampMap); // Build map of partition => offset final Map<TopicPartition, Long> partitionOffsetMap = new HashMap<>(); for (Map.Entry<TopicPartition, OffsetAndTimestamp> entry: offsetMap.entrySet()) { partitionOffsetMap.put(entry.getKey(), entry.getValue().offset()); } // Now lets seek to those offsets seek(partitionOffsetMap); }
private long findClosestOffsetJustBeforeTimestamp(KafkaConsumer<byte[], byte[]> consumer, KafkaTopic kafkaTopic, int partition, long timestamp) { long endOffset = getEndingOffset(consumer, kafkaTopic, partition); TopicPartition topicPartition = new TopicPartition(kafkaTopic.name().asString(), partition); return Optional.ofNullable(consumer.offsetsForTimes(Collections.singletonMap(topicPartition, timestamp)).get(topicPartition)) .orElse(new OffsetAndTimestamp(endOffset, timestamp)).offset(); }
/** * Seek consumer to specific timestamp * @param timestamp Unix timestamp in milliseconds to seek to. */ public ConsumerState seek(final long timestamp) { // Find offsets for timestamp final Map<TopicPartition, Long> timestampMap = new HashMap<>(); for (final TopicPartition topicPartition: getAllPartitions()) { timestampMap.put(topicPartition, timestamp); } final Map<TopicPartition, OffsetAndTimestamp> offsetMap = kafkaConsumer.offsetsForTimes(timestampMap); // Build map of partition => offset final Map<Integer, Long> partitionOffsetMap = new HashMap<>(); for (Map.Entry<TopicPartition, OffsetAndTimestamp> entry: offsetMap.entrySet()) { partitionOffsetMap.put(entry.getKey().partition(), entry.getValue().offset()); } // Now lets seek to those offsets return seek(partitionOffsetMap); }
@Override public LogOffset offsetForTimestamp(LogPartition partition, long timestamp) { TopicPartition topicPartition = new TopicPartition(ns.getTopicName(partition.name()), partition.partition()); Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes = consumer.offsetsForTimes( Collections.singletonMap(topicPartition, timestamp)); if (offsetsForTimes.size() == 1) { OffsetAndTimestamp offsetAndTimestamp = offsetsForTimes.get(topicPartition); if (offsetAndTimestamp != null) { return new LogOffsetImpl(partition, offsetAndTimestamp.offset()); } } return null; }
@Override protected Map<KafkaTopicPartition, Long> fetchOffsetsWithTimestamp( Collection<KafkaTopicPartition> partitions, long timestamp) { Map<TopicPartition, Long> partitionOffsetsRequest = new HashMap<>(partitions.size()); for (KafkaTopicPartition partition : partitions) { partitionOffsetsRequest.put( new TopicPartition(partition.getTopic(), partition.getPartition()), timestamp); } // use a short-lived consumer to fetch the offsets; // this is ok because this is a one-time operation that happens only on startup KafkaConsumer<?, ?> consumer = new KafkaConsumer(properties); Map<KafkaTopicPartition, Long> result = new HashMap<>(partitions.size()); for (Map.Entry<TopicPartition, OffsetAndTimestamp> partitionToOffset : consumer.offsetsForTimes(partitionOffsetsRequest).entrySet()) { result.put( new KafkaTopicPartition(partitionToOffset.getKey().topic(), partitionToOffset.getKey().partition()), (partitionToOffset.getValue() == null) ? null : partitionToOffset.getValue().offset()); } consumer.close(); return result; } }
protected void positionConsumer(final Map<Integer, TopicPartition> partitions, final MessageFilter filter, final KafkaConsumer<Object, Object> consumer,Map<Integer, Long> partitionOffsets) { if (filter.getFrom() == null) { // no start time, position to the beginning consumer.seekToBeginning(partitions.values()); } else { // position to time offsets final Map<TopicPartition, Long> query = new HashMap<>(); final long out = Date.from(filter.getFrom().atZone(ZoneId.of("UTC")).toInstant()).getTime(); for (final TopicPartition topicPartition : partitions.values()) { query.put(topicPartition, out); } final Map<TopicPartition, OffsetAndTimestamp> result = consumer.offsetsForTimes(query); result.entrySet().stream().forEach(entry -> { if (entry.getValue() != null) { Long offset = entry.getValue().offset(); TopicPartition partition = entry.getKey(); consumer.seek(partition, offset); partitionOffsets.put(partition.partition(), offset); }else { TopicPartition partition = entry.getKey(); consumer.seekToEnd(Collections.singleton(partition)); partitionOffsets.put(partition.partition(), consumer.position(partition)); } }); } }
private static Map<KafkaTopicPartition, Long> buildOffsetByTime(Properties props, ParameterTool parameterTool, Long time) { props.setProperty("group.id", "query_time_" + time); KafkaConsumer consumer = new KafkaConsumer(props); List<PartitionInfo> partitionsFor = consumer.partitionsFor(parameterTool.getRequired(PropertiesConstants.METRICS_TOPIC)); Map<TopicPartition, Long> partitionInfoLongMap = new HashMap<>(); for (PartitionInfo partitionInfo : partitionsFor) { partitionInfoLongMap.put(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()), time); } Map<TopicPartition, OffsetAndTimestamp> offsetResult = consumer.offsetsForTimes(partitionInfoLongMap); Map<KafkaTopicPartition, Long> partitionOffset = new HashMap<>(); offsetResult.forEach((key, value) -> partitionOffset.put(new KafkaTopicPartition(key.topic(), key.partition()), value.offset())); consumer.close(); return partitionOffset; }
@Override protected Map<KafkaTopicPartition, Long> fetchOffsetsWithTimestamp( Collection<KafkaTopicPartition> partitions, long timestamp) { Map<TopicPartition, Long> partitionOffsetsRequest = new HashMap<>(partitions.size()); for (KafkaTopicPartition partition : partitions) { partitionOffsetsRequest.put( new TopicPartition(partition.getTopic(), partition.getPartition()), timestamp); } // use a short-lived consumer to fetch the offsets; // this is ok because this is a one-time operation that happens only on startup KafkaConsumer<?, ?> consumer = createKafkaConsumer(properties); Map<KafkaTopicPartition, Long> result = new HashMap<>(partitions.size()); for (Map.Entry<TopicPartition, OffsetAndTimestamp> partitionToOffset : consumer.offsetsForTimes(partitionOffsetsRequest).entrySet()) { result.put( new KafkaTopicPartition(partitionToOffset.getKey().topic(), partitionToOffset.getKey().partition()), (partitionToOffset.getValue() == null) ? null : partitionToOffset.getValue().offset()); } consumer.close(); return result; }
Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestamp = consumer.offsetsForTimes(timePartitionsMap); return Maps.toMap(fullHouse.keySet(), tp -> { KafkaInputSplit existing = fullHouse.get(tp);