@Override public Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> getSSPMetadata( Set<SystemStreamPartition> ssps) { LOG.info("Fetching SSP metadata for: {}", ssps); List<TopicPartition> topicPartitions = ssps.stream() .map(ssp -> new TopicPartition(ssp.getStream(), ssp.getPartition().getPartitionId())) .collect(Collectors.toList()); OffsetsMaps topicPartitionsMetadata = fetchTopicPartitionsMetadata(topicPartitions); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> sspToSSPMetadata = new HashMap<>(); for (SystemStreamPartition ssp : ssps) { String oldestOffset = topicPartitionsMetadata.getOldestOffsets().get(ssp); String newestOffset = topicPartitionsMetadata.getNewestOffsets().get(ssp); String upcomingOffset = topicPartitionsMetadata.getUpcomingOffsets().get(ssp); sspToSSPMetadata.put(ssp, new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset)); } return sspToSSPMetadata; }
@Override public Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> getSSPMetadata( Set<SystemStreamPartition> ssps) { LOG.info("Fetching SSP metadata for: {}", ssps); List<TopicPartition> topicPartitions = ssps.stream() .map(ssp -> new TopicPartition(ssp.getStream(), ssp.getPartition().getPartitionId())) .collect(Collectors.toList()); OffsetsMaps topicPartitionsMetadata = fetchTopicPartitionsMetadata(topicPartitions); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> sspToSSPMetadata = new HashMap<>(); for (SystemStreamPartition ssp : ssps) { String oldestOffset = topicPartitionsMetadata.getOldestOffsets().get(ssp); String newestOffset = topicPartitionsMetadata.getNewestOffsets().get(ssp); String upcomingOffset = topicPartitionsMetadata.getUpcomingOffsets().get(ssp); sspToSSPMetadata.put(ssp, new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset)); } return sspToSSPMetadata; }
@Override public Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> getSSPMetadata( Set<SystemStreamPartition> ssps) { LOG.info("Fetching SSP metadata for: {}", ssps); List<TopicPartition> topicPartitions = ssps.stream() .map(ssp -> new TopicPartition(ssp.getStream(), ssp.getPartition().getPartitionId())) .collect(Collectors.toList()); OffsetsMaps topicPartitionsMetadata = fetchTopicPartitionsMetadata(topicPartitions); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> sspToSSPMetadata = new HashMap<>(); for (SystemStreamPartition ssp : ssps) { String oldestOffset = topicPartitionsMetadata.getOldestOffsets().get(ssp); String newestOffset = topicPartitionsMetadata.getNewestOffsets().get(ssp); String upcomingOffset = topicPartitionsMetadata.getUpcomingOffsets().get(ssp); sspToSSPMetadata.put(ssp, new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset)); } return sspToSSPMetadata; }
/** * Fetch SystemStreamMetadata for each topic with the consumer * @param topics set of topics to get metadata info for * @return map of topic to SystemStreamMetadata */ private Map<String, SystemStreamMetadata> fetchSystemStreamMetadata(Set<String> topics) { Map<SystemStreamPartition, String> allOldestOffsets = new HashMap<>(); Map<SystemStreamPartition, String> allNewestOffsets = new HashMap<>(); Map<SystemStreamPartition, String> allUpcomingOffsets = new HashMap<>(); LOG.info("Fetching SystemStreamMetadata for topics {} on system {}", topics, systemName); topics.forEach(topic -> { List<PartitionInfo> partitionInfos = metadataConsumer.partitionsFor(topic); if (partitionInfos == null) { String msg = String.format("Partition info not(yet?) available for system %s topic %s", systemName, topic); throw new SamzaException(msg); } List<TopicPartition> topicPartitions = partitionInfos.stream() .map(partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition())) .collect(Collectors.toList()); OffsetsMaps offsetsForTopic = fetchTopicPartitionsMetadata(topicPartitions); allOldestOffsets.putAll(offsetsForTopic.getOldestOffsets()); allNewestOffsets.putAll(offsetsForTopic.getNewestOffsets()); allUpcomingOffsets.putAll(offsetsForTopic.getUpcomingOffsets()); }); scala.collection.immutable.Map<String, SystemStreamMetadata> result = KafkaSystemAdminUtilsScala.assembleMetadata(ScalaJavaUtil.toScalaMap(allOldestOffsets), ScalaJavaUtil.toScalaMap(allNewestOffsets), ScalaJavaUtil.toScalaMap(allUpcomingOffsets)); LOG.debug("assembled SystemStreamMetadata is: {}", result); return JavaConverters.mapAsJavaMapConverter(result).asJava(); }
/** * Fetch SystemStreamMetadata for each topic with the consumer * @param topics set of topics to get metadata info for * @return map of topic to SystemStreamMetadata */ private Map<String, SystemStreamMetadata> fetchSystemStreamMetadata(Set<String> topics) { Map<SystemStreamPartition, String> allOldestOffsets = new HashMap<>(); Map<SystemStreamPartition, String> allNewestOffsets = new HashMap<>(); Map<SystemStreamPartition, String> allUpcomingOffsets = new HashMap<>(); LOG.info("Fetching SystemStreamMetadata for topics {} on system {}", topics, systemName); topics.forEach(topic -> { List<PartitionInfo> partitionInfos = metadataConsumer.partitionsFor(topic); if (partitionInfos == null) { String msg = String.format("Partition info not(yet?) available for system %s topic %s", systemName, topic); throw new SamzaException(msg); } List<TopicPartition> topicPartitions = partitionInfos.stream() .map(partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition())) .collect(Collectors.toList()); OffsetsMaps offsetsForTopic = fetchTopicPartitionsMetadata(topicPartitions); allOldestOffsets.putAll(offsetsForTopic.getOldestOffsets()); allNewestOffsets.putAll(offsetsForTopic.getNewestOffsets()); allUpcomingOffsets.putAll(offsetsForTopic.getUpcomingOffsets()); }); scala.collection.immutable.Map<String, SystemStreamMetadata> result = KafkaSystemAdminUtilsScala.assembleMetadata(ScalaJavaUtil.toScalaMap(allOldestOffsets), ScalaJavaUtil.toScalaMap(allNewestOffsets), ScalaJavaUtil.toScalaMap(allUpcomingOffsets)); LOG.debug("assembled SystemStreamMetadata is: {}", result); return JavaConverters.mapAsJavaMapConverter(result).asJava(); }
/** * Fetch SystemStreamMetadata for each topic with the consumer * @param topics set of topics to get metadata info for * @return map of topic to SystemStreamMetadata */ private Map<String, SystemStreamMetadata> fetchSystemStreamMetadata(Set<String> topics) { Map<SystemStreamPartition, String> allOldestOffsets = new HashMap<>(); Map<SystemStreamPartition, String> allNewestOffsets = new HashMap<>(); Map<SystemStreamPartition, String> allUpcomingOffsets = new HashMap<>(); LOG.info("Fetching SystemStreamMetadata for topics {} on system {}", topics, systemName); topics.forEach(topic -> { List<PartitionInfo> partitionInfos = metadataConsumer.partitionsFor(topic); if (partitionInfos == null) { String msg = String.format("Partition info not(yet?) available for system %s topic %s", systemName, topic); throw new SamzaException(msg); } List<TopicPartition> topicPartitions = partitionInfos.stream() .map(partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition())) .collect(Collectors.toList()); OffsetsMaps offsetsForTopic = fetchTopicPartitionsMetadata(topicPartitions); allOldestOffsets.putAll(offsetsForTopic.getOldestOffsets()); allNewestOffsets.putAll(offsetsForTopic.getNewestOffsets()); allUpcomingOffsets.putAll(offsetsForTopic.getUpcomingOffsets()); }); scala.collection.immutable.Map<String, SystemStreamMetadata> result = KafkaSystemAdminUtilsScala.assembleMetadata(ScalaJavaUtil.toScalaMap(allOldestOffsets), ScalaJavaUtil.toScalaMap(allNewestOffsets), ScalaJavaUtil.toScalaMap(allUpcomingOffsets)); LOG.debug("assembled SystemStreamMetadata is: {}", result); return JavaConverters.mapAsJavaMapConverter(result).asJava(); }