/** * The topic the record was appended to */ public String topic() { return this.topicPartition.topic(); }
private ConsumerPair removeMovementRecordOfPartition(TopicPartition partition) { ConsumerPair pair = partitionMovements.remove(partition); String topic = partition.topic(); Map<ConsumerPair, Set<TopicPartition>> partitionMovementsForThisTopic = partitionMovementsByTopic.get(topic); partitionMovementsForThisTopic.get(pair).remove(partition); if (partitionMovementsForThisTopic.get(pair).isEmpty()) partitionMovementsForThisTopic.remove(pair); if (partitionMovementsByTopic.get(topic).isEmpty()) partitionMovementsByTopic.remove(topic); return pair; }
private void addPartitionMovementRecord(TopicPartition partition, ConsumerPair pair) { partitionMovements.put(partition, pair); String topic = partition.topic(); if (!partitionMovementsByTopic.containsKey(topic)) partitionMovementsByTopic.put(topic, new HashMap<ConsumerPair, Set<TopicPartition>>()); Map<ConsumerPair, Set<TopicPartition>> partitionMovementsForThisTopic = partitionMovementsByTopic.get(topic); if (!partitionMovementsForThisTopic.containsKey(pair)) partitionMovementsForThisTopic.put(pair, new HashSet<TopicPartition>()); partitionMovementsForThisTopic.get(pair).add(partition); }
synchronized void retainTopics(Collection<String> topics) { metadataByPartition.entrySet().removeIf(entry -> !topics.contains(entry.getKey().topic())); unauthorizedTopics.retainAll(topics); invalidTopics.retainAll(topics); computeClusterView(); }
/** * Get just the records for the given topic */ public Iterable<ConsumerRecord<K, V>> records(String topic) { if (topic == null) throw new IllegalArgumentException("Topic must be non-null."); List<List<ConsumerRecord<K, V>>> recs = new ArrayList<>(); for (Map.Entry<TopicPartition, List<ConsumerRecord<K, V>>> entry : records.entrySet()) { if (entry.getKey().topic().equals(topic)) recs.add(entry.getValue()); } return new ConcatenatedIterable<>(recs); }
private void update(Map<TopicPartition, S> partitionToState) { LinkedHashMap<String, List<TopicPartition>> topicToPartitions = new LinkedHashMap<>(); for (TopicPartition tp : partitionToState.keySet()) { List<TopicPartition> partitions = topicToPartitions.computeIfAbsent(tp.topic(), k -> new ArrayList<>()); partitions.add(tp); } for (Map.Entry<String, List<TopicPartition>> entry : topicToPartitions.entrySet()) { for (TopicPartition tp : entry.getValue()) { S state = partitionToState.get(tp); map.put(tp, state); } } }
/** * Group a list of partitions by the topic name. * * @param partitions The partitions to collect * @return partitions per topic */ public static Map<String, List<Integer>> groupPartitionsByTopic(Collection<TopicPartition> partitions) { Map<String, List<Integer>> partitionsByTopic = new HashMap<>(); for (TopicPartition tp : partitions) { String topic = tp.topic(); List<Integer> topicData = partitionsByTopic.computeIfAbsent(topic, t -> new ArrayList<>()); topicData.add(tp.partition()); } return partitionsByTopic; }
private TopicPartition getTheActualPartitionToBeMoved(TopicPartition partition, String oldConsumer, String newConsumer) { String topic = partition.topic(); if (!partitionMovementsByTopic.containsKey(topic)) return partition; if (partitionMovements.containsKey(partition)) { // this partition has previously moved assert oldConsumer.equals(partitionMovements.get(partition).dstMemberId); oldConsumer = partitionMovements.get(partition).srcMemberId; } Map<ConsumerPair, Set<TopicPartition>> partitionMovementsForThisTopic = partitionMovementsByTopic.get(topic); ConsumerPair reversePair = new ConsumerPair(newConsumer, oldConsumer); if (!partitionMovementsForThisTopic.containsKey(reversePair)) return partition; return partitionMovementsForThisTopic.get(reversePair).iterator().next(); }
public static <T> List<TopicAndPartitionData<T>> batchByTopic(Iterator<Map.Entry<TopicPartition, T>> iter) { List<TopicAndPartitionData<T>> topics = new ArrayList<>(); while (iter.hasNext()) { Map.Entry<TopicPartition, T> topicEntry = iter.next(); String topic = topicEntry.getKey().topic(); int partition = topicEntry.getKey().partition(); T partitionData = topicEntry.getValue(); if (topics.isEmpty() || !topics.get(topics.size() - 1).topic.equals(topic)) topics.add(new TopicAndPartitionData<T>(topic)); topics.get(topics.size() - 1).partitions.put(partition, partitionData); } return topics; } }
/** * Clear the buffered data which are not a part of newly assigned topics * * @param assignedTopics newly assigned topics */ public void clearBufferedDataForUnassignedTopics(Collection<String> assignedTopics) { Set<TopicPartition> currentTopicPartitions = new HashSet<>(); for (TopicPartition tp : subscriptions.assignedPartitions()) { if (assignedTopics.contains(tp.topic())) { currentTopicPartitions.add(tp); } } clearBufferedDataForUnassignedPartitions(currentTopicPartitions); }
@Override public Set<StreamPartition<Integer>> getAssignment() { Set<TopicPartition> topicPartitions = consumer.assignment(); return topicPartitions .stream() .map(e -> new StreamPartition<>(e.topic(), e.partition())) .collect(Collectors.toSet()); }
@Override public Map<String, List<TopicPartition>> assign(Map<String, Integer> partitionsPerTopic, Map<String, Subscription> subscriptions) { Map<String, List<TopicPartition>> assignment = new HashMap<>(); for (String memberId : subscriptions.keySet()) assignment.put(memberId, new ArrayList<>()); CircularIterator<String> assigner = new CircularIterator<>(Utils.sorted(subscriptions.keySet())); for (TopicPartition partition : allPartitionsSorted(partitionsPerTopic, subscriptions)) { final String topic = partition.topic(); while (!subscriptions.get(assigner.peek()).topics().contains(topic)) assigner.next(); assignment.get(assigner.next()).add(partition); } return assignment; }
private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) { this.accumulator.reenqueue(batch, currentTimeMs); maybeRemoveFromInflightBatches(batch); this.sensors.recordRetries(batch.topicPartition.topic(), batch.recordCount); }
@Override protected Struct toStruct(short version) { Struct struct = new Struct(ApiKeys.CONTROLLED_SHUTDOWN.responseSchema(version)); struct.set(ERROR_CODE, error.code()); List<Struct> partitionsRemainingList = new ArrayList<>(partitionsRemaining.size()); for (TopicPartition topicPartition : partitionsRemaining) { Struct topicPartitionStruct = struct.instance(PARTITIONS_REMAINING_KEY_NAME); topicPartitionStruct.set(TOPIC_NAME, topicPartition.topic()); topicPartitionStruct.set(PARTITION_ID, topicPartition.partition()); partitionsRemainingList.add(topicPartitionStruct); } struct.set(PARTITIONS_REMAINING_KEY_NAME, partitionsRemainingList.toArray()); return struct; } }
public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, long createdMs, boolean isSplitBatch) { this.createdMs = createdMs; this.lastAttemptMs = createdMs; this.recordsBuilder = recordsBuilder; this.topicPartition = tp; this.lastAppendTime = createdMs; this.produceFuture = new ProduceRequestResult(topicPartition); this.retry = false; this.isSplitBatch = isSplitBatch; float compressionRatioEstimation = CompressionRatioEstimator.estimation(topicPartition.topic(), recordsBuilder.compressionType()); recordsBuilder.setEstimatedCompressionRatio(compressionRatioEstimation); }
private void checkValues(TopicPartition deSerTP) { //assert deserialized values are same as original assertEquals("partition number should be " + partNum + " but got " + deSerTP.partition(), partNum, deSerTP.partition()); assertEquals("topic should be " + topicName + " but got " + deSerTP.topic(), topicName, deSerTP.topic()); }
public void close() { recordsBuilder.close(); if (!recordsBuilder.isControlBatch()) { CompressionRatioEstimator.updateEstimation(topicPartition.topic(), recordsBuilder.compressionType(), (float) recordsBuilder.compressionRatio()); } reopened = false; }
private void recordPartitionLead(TopicPartition tp, long lead) { this.recordsFetchLead.record(lead); String name = partitionLeadMetricName(tp); Sensor recordsLead = this.metrics.getSensor(name); if (recordsLead == null) { Map<String, String> metricTags = new HashMap<>(2); metricTags.put("topic", tp.topic().replace('.', '_')); metricTags.put("partition", String.valueOf(tp.partition())); recordsLead = this.metrics.sensor(name); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLead, metricTags), new Value()); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadMin, metricTags), new Min()); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadAvg, metricTags), new Avg()); } recordsLead.record(lead); }
private void recordPartitionLag(TopicPartition tp, long lag) { this.recordsFetchLag.record(lag); String name = partitionLagMetricName(tp); Sensor recordsLag = this.metrics.getSensor(name); if (recordsLag == null) { Map<String, String> metricTags = new HashMap<>(2); metricTags.put("topic", tp.topic().replace('.', '_')); metricTags.put("partition", String.valueOf(tp.partition())); recordsLag = this.metrics.sensor(name); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLag, metricTags), new Value()); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLagMax, metricTags), new Max()); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLagAvg, metricTags), new Avg()); } recordsLag.record(lag); }