private static Set<InstanceTopicPartitionHolder> balanceAssignment( TreeSet<InstanceTopicPartitionHolder> orderedSet) { while (!isAssignmentBalanced(orderedSet)) { InstanceTopicPartitionHolder lowestInstance = orderedSet.pollFirst(); InstanceTopicPartitionHolder highestInstance = orderedSet.pollLast(); TopicPartition tpi = highestInstance.getServingTopicPartitionSet().iterator().next(); highestInstance.removeTopicPartition(tpi); lowestInstance.addTopicPartition(tpi); orderedSet.add(lowestInstance); orderedSet.add(highestInstance); } return orderedSet; }
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<String, CustomModeISBuilder>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<String, IdealState>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
boolean isStuckInstance = false; for (TopicPartition tp : allStuckPartitions) { if (itph.getServingTopicPartitionSet().contains(tp)) { isStuckInstance = true; List<Long> moveHistory = _movePartitionHistoryMap.get(tp);
InstanceTopicPartitionHolder itph = iter.next(); LOGGER.info("Move workload from instance " + itph.getInstanceName() + " to become idle: " + itph.getServingTopicPartitionSet()); tpiNeedsToBeAssigned.addAll(itph.getServingTopicPartitionSet()); iter.remove();
for (TopicPartition tp : instance.getServingTopicPartitionSet()) { long lag = getLagTime(tp); if (lag > 0) { } else { boolean hasLag = false; for (TopicPartition tp : instance.getServingTopicPartitionSet()) { if (lagTimeMap.containsKey(tp)) { hasLag = true; for (TopicPartition tp : instance.getServingTopicPartitionSet()) { if (!lagTimeMap.containsKey(tp)) { instance.removeTopicPartition(tp);
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
for (TopicPartition tp : instance.getServingTopicPartitionSet()) { double tpw = workloadRetriever.topicWorkload(tp.getTopic()).getBytesPerSecondPerPartition(); totalWorkload += tpw; instanceMapJson.getJSONArray(name).add(tp.getTopic() + "." + tp.getPartition() + ":" + Math.round(tpw)); instanceMapJson.getJSONArray(name).add("TOTALWORKLOAD." + instance.getServingTopicPartitionSet().size() + ":" + Math.round(totalWorkload));
for (InstanceTopicPartitionHolder instance : orderedInstances) { int partitionCount = 0; for (TopicPartition tp : instance.getServingTopicPartitionSet()) { double weight = weighter.partitionWeight(tp); if (weight > 0) { for (InstanceTopicPartitionHolder instance : orderedInstances) { if (instancesPartitionsCount.get(instance.getInstanceName()) == 1) { for (TopicPartition tp : instance.getServingTopicPartitionSet()) { if (weighter.partitionWeight(tp) > 0) { double weight = weighter.partitionWeight(tp); workerWorkload.getMsgsPerSecond() - maxWorkload.getMsgsPerSecond()); TopicWorkload workloadToRemove = new TopicWorkload(0, 0, 0); List<TopicPartition> partitions = new ArrayList<>(highest.getServingTopicPartitionSet());