private static boolean replicaListChanged(PartitionInfo prevPartInfo, PartitionInfo currPartInfo) { if (prevPartInfo.replicas().length != currPartInfo.replicas().length) { return true; } for (int i = 0; i < prevPartInfo.replicas().length; i++) { if (prevPartInfo.replicas()[i].id() != currPartInfo.replicas()[i].id()) { return true; } } return false; }
static int getReplicationFactor(List<PartitionInfo> partitionInfoList) { if (partitionInfoList.isEmpty()) throw new RuntimeException("Partition list is empty"); int replicationFactor = partitionInfoList.get(0).replicas().length; for (PartitionInfo partitionInfo : partitionInfoList) { if (replicationFactor != partitionInfo.replicas().length) { String topic = partitionInfoList.get(0).topic(); LOG.warn("Partitions of the topic " + topic + " have different replication factor"); return -1; } } return replicationFactor; }
static boolean someBrokerNotPreferredLeader(List<PartitionInfo> partitionInfoList, Collection<Broker> brokers) { Set<Integer> brokersNotPreferredLeader = new HashSet<>(brokers.size()); for (Broker broker: brokers) brokersNotPreferredLeader.add(broker.id()); for (PartitionInfo partitionInfo : partitionInfoList) brokersNotPreferredLeader.remove(partitionInfo.replicas()[0].id()); return !brokersNotPreferredLeader.isEmpty(); }
private Set<Integer> brokersWithPartitions(Cluster kafkaCluster) { Set<Integer> allBrokers = new HashSet<>(); for (String topic : kafkaCluster.topics()) { for (PartitionInfo pi : kafkaCluster.partitionsForTopic(topic)) { for (Node node : pi.replicas()) { allBrokers.add(node.id()); } } } return allBrokers; }
/** * Check if the partition is currently under replicated. * @param cluster The current cluster state. * @param tp The topic partition to check. * @return True if the partition is currently under replicated. */ public static boolean isPartitionUnderReplicated(Cluster cluster, TopicPartition tp) { PartitionInfo partitionInfo = cluster.partition(tp); return partitionInfo.inSyncReplicas().length != partitionInfo.replicas().length; } }
private List<Object> getJsonPartitions(Set<PartitionInfo> partitions) { List<Object> partitionList = new ArrayList<>(); for (PartitionInfo partitionInfo : partitions) { Set<Integer> replicas = Arrays.stream(partitionInfo.replicas()).map(Node::id).collect(Collectors.toSet()); Set<Integer> inSyncReplicas = Arrays.stream(partitionInfo.inSyncReplicas()).map(Node::id).collect(Collectors.toSet()); Set<Integer> outOfSyncReplicas = new HashSet<>(replicas); outOfSyncReplicas.removeAll(inSyncReplicas); Map<String, Object> recordMap = new HashMap<>(); recordMap.put(TOPIC, partitionInfo.topic()); recordMap.put(PARTITION, partitionInfo.partition()); recordMap.put(LEADER, partitionInfo.leader() == null ? -1 : partitionInfo.leader().id()); recordMap.put(REPLICAS, replicas); recordMap.put(IN_SYNC, inSyncReplicas); recordMap.put(OUT_OF_SYNC, outOfSyncReplicas); partitionList.add(recordMap); } return partitionList; }
private void writeKafkaClusterState(StringBuilder sb, SortedSet<PartitionInfo> partitions, int topicNameLength) { for (PartitionInfo partitionInfo : partitions) { Set<String> replicas = Arrays.stream(partitionInfo.replicas()).map(Node::idString).collect(Collectors.toSet()); Set<String> inSyncReplicas = Arrays.stream(partitionInfo.inSyncReplicas()).map(Node::idString).collect(Collectors.toSet()); Set<String> outOfSyncReplicas = new HashSet<>(replicas); outOfSyncReplicas.removeAll(inSyncReplicas); sb.append(String.format("%" + topicNameLength + "s%10s%10s%40s%40s%30s%n", partitionInfo.topic(), partitionInfo.partition(), partitionInfo.leader() == null ? -1 : partitionInfo.leader().id(), replicas, inSyncReplicas, outOfSyncReplicas)); } }
for (PartitionInfo partitionInfo : partitionInfos) { TopicPartitionInfo topicPartitionInfo = new TopicPartitionInfo( partitionInfo.partition(), leader(partitionInfo), Arrays.asList(partitionInfo.replicas()), Arrays.asList(partitionInfo.inSyncReplicas())); partitions.add(topicPartitionInfo);
Arrays.stream(partitionInfo.replicas()).map(Node::id).collect(Collectors.toSet()); Set<Integer> inSyncReplicas = Arrays.stream(partitionInfo.inSyncReplicas()).map(Node::id).collect(Collectors.toSet());
/** * Gather the Kafka partition state within the given under replicated, offline, and other partitions (if verbose). * * @param underReplicatedPartitions state of under replicated partitions. * @param offlinePartitions state of offline partitions. * @param otherPartitions state of partitions other than offline or urp. * @param verbose true if requested to gather state of partitions other than offline or urp. */ private void populateKafkaPartitionState(Set<PartitionInfo> underReplicatedPartitions, Set<PartitionInfo> offlinePartitions, Set<PartitionInfo> otherPartitions, boolean verbose) { for (String topic : _kafkaCluster.topics()) { for (PartitionInfo partitionInfo : _kafkaCluster.partitionsForTopic(topic)) { boolean isURP = partitionInfo.inSyncReplicas().length != partitionInfo.replicas().length; if (isURP || verbose) { boolean isOffline = partitionInfo.inSyncReplicas().length == 0; if (isOffline) { offlinePartitions.add(partitionInfo); } else if (isURP) { underReplicatedPartitions.add(partitionInfo); } else { // verbose -- other otherPartitions.add(partitionInfo); } } } } }
/** * For each proposal, create a replica action task if there is a need for moving replica(s) to reach expected final proposal state. * * @param proposals Execution proposals. * @param cluster Kafka cluster state. */ private void maybeAddReplicaMovementTasks(Collection<ExecutionProposal> proposals, Cluster cluster) { for (ExecutionProposal proposal : proposals) { TopicPartition tp = proposal.topicPartition(); PartitionInfo partitionInfo = cluster.partition(tp); if (partitionInfo == null) { LOG.trace("Ignored the attempt to move non-existing partition for topic partition: {}", tp); continue; } if (!proposal.isCompletedSuccessfully(partitionInfo.replicas())) { long replicaActionExecutionId = _executionId++; ExecutionTask executionTask = new ExecutionTask(replicaActionExecutionId, proposal, REPLICA_ACTION); _remainingReplicaMovements.add(executionTask); _remainingDataToMove += proposal.dataToMoveInMB(); LOG.trace("Added action {} as replica proposal {}", replicaActionExecutionId, proposal); } } _partMoveTaskByBrokerId = _replicaMovementTaskStrategy.applyStrategy(_remainingReplicaMovements, cluster); }
/** * For a replica action, the completion depends on the task state: * IN_PROGRESS: when the current replica list is the same as the new replica list. * ABORTING: done when the current replica list is the same as the old replica list. Due to race condition, * we also consider it done if the current replica list is the same as the new replica list. * DEAD: always considered as done because we neither move forward or rollback. * * There should be no other task state seen here. */ private boolean isReplicaActionDone(Cluster cluster, TopicPartition tp, ExecutionTask task) { Node[] currentOrderedReplicas = cluster.partition(tp).replicas(); switch (task.state()) { case IN_PROGRESS: return task.proposal().isCompletedSuccessfully(currentOrderedReplicas); case ABORTING: return task.proposal().isAborted(currentOrderedReplicas); case DEAD: return true; default: throw new IllegalStateException("Should never be here. State " + task.state()); } }
if (partitionInfo.replicas().length > 1 && partitionInfo.leader() != null && partitionInfo.leader().id() == brokerId) {
/** * When the replica is a leader replica, we need to fill in the replication bytes out if it has not been filled in * yet. This is because currently Kafka does not report this metric. We simply use the leader bytes in rate multiplied * by the number of followers as the replication bytes out rate. The assumption is that all the followers will * eventually keep up with the leader. * * We only fill in the replication bytes out rate when creating the cluster model because the replication factor * may have changed since the time the PartitionMetricSample was created. * * @param aggregatedMetricValues the {@link AggregatedMetricValues} for the leader replica. * @param info the partition info for the partition. * @return the {@link AggregatedMetricValues} with the replication bytes out rate filled in. */ private AggregatedMetricValues fillInReplicationBytesOut(AggregatedMetricValues aggregatedMetricValues, PartitionInfo info) { int numFollowers = info.replicas().length - 1; int leaderBytesInRateId = KafkaMetricDef.commonMetricDefId(KafkaMetricDef.LEADER_BYTES_IN); int replicationBytesOutRateId = KafkaMetricDef.commonMetricDefId(KafkaMetricDef.REPLICATION_BYTES_OUT_RATE); MetricValues leaderBytesInRate = aggregatedMetricValues.valuesFor(leaderBytesInRateId); MetricValues replicationBytesOutRate = aggregatedMetricValues.valuesFor(replicationBytesOutRateId); // If the replication bytes out rate is already reported, update it. Otherwise add a new MetricValues. if (replicationBytesOutRate == null) { replicationBytesOutRate = new MetricValues(leaderBytesInRate.length()); aggregatedMetricValues.add(replicationBytesOutRateId, replicationBytesOutRate); } for (int i = 0; i < leaderBytesInRate.length(); i++) { replicationBytesOutRate.set(i, leaderBytesInRate.get(i) * numFollowers); } return aggregatedMetricValues; }
@Override public KafkaTopic getTopic(final String name) { KafkaTopic kafkaTopic = null; if (listTopics().contains(name)) { try (Consumer<String, String> consumer = kafkaConsumerFactory.createConsumer()) { final List<PartitionInfo> partitionInfos = consumer.partitionsFor(name); if (partitionInfos.size() > 0) { final PartitionInfo partitionInfo = partitionInfos.get(0); kafkaTopic = new KafkaTopic(); kafkaTopic.setName(name); kafkaTopic.setNumPartitions(partitionInfos.size()); kafkaTopic.setReplicationFactor(partitionInfo.replicas().length); } } } return kafkaTopic; }
/** * Get the under replicated nodes from PartitionInfo */ public static Set<Node> getNotInSyncBrokers(PartitionInfo partitionInfo) { if (partitionInfo.inSyncReplicas().length == partitionInfo.replicas().length) { return new HashSet<>(); } Set<Node> nodes = new HashSet<>(Arrays.asList(partitionInfo.replicas())); for (Node node : partitionInfo.inSyncReplicas()) { nodes.remove(node); } return nodes; }
/** * Get the under replicated nodes from PartitionInfo */ public static Set<Node> getNotInSyncBrokers(PartitionInfo partitionInfo) { if (partitionInfo.inSyncReplicas().length == partitionInfo.replicas().length) { return new HashSet<>(); } Set<Node> nodes = new HashSet<>(Arrays.asList(partitionInfo.replicas())); for (Node node : partitionInfo.inSyncReplicas()) { nodes.remove(node); } return nodes; }
/** * Get the under replicated nodes from PartitionInfo */ public static Set<Integer> getOutOfSyncReplicas(PartitionInfo partitionInfo) { if (partitionInfo.inSyncReplicas().length == partitionInfo.replicas().length) { return new HashSet<>(); } Set<Node> nodes = new HashSet<>(Arrays.asList(partitionInfo.replicas())); for (Node node : partitionInfo.inSyncReplicas()) { nodes.remove(node); } return nodes.stream().map(nd -> nd.id()).collect(Collectors.toSet()); }
public Map<Integer, Long> countPartition(String topic) { KafkaConsumer consumer = kafkaUtils.createNewConsumer(); List<PartitionInfo> piList = consumer.partitionsFor(topic); Map<Integer, Long> result = piList .stream() .flatMap(pi -> Arrays.stream(pi.replicas())) .map(node -> node.id()) .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); consumer.close(); return result; }