@Override public int compare(InstanceTopicPartitionHolder o1, InstanceTopicPartitionHolder o2) { int size1 = (o1 == null) ? -1 : o1.getNumServingTopicPartitions(); int size2 = (o2 == null) ? -1 : o2.getNumServingTopicPartitions(); if (size1 != size2) { return size1 - size2; } else { return o1.getInstanceName().compareTo(o2.getInstanceName()); } } };
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<String, CustomModeISBuilder>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<String, IdealState>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
public static IdealState expandCustomRebalanceModeIdealStateFor(IdealState oldIdealState, String topicName, int newNumTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(newNumTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(newNumTopicPartitions); int numOldPartitions = oldIdealState.getNumPartitions(); for (int i = 0; i < numOldPartitions; ++i) { String partitionName = Integer.toString(i); try { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } catch (Exception e) { // No worker added into the cluster. } } for (int i = numOldPartitions; i < newNumTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } return customModeIdealStateBuilder.build(); }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); if (liveInstance != null) { customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { synchronized (instanceToNumServingTopicPartitionMap) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
} else if (moveHistory.size() >= _maxStuckPartitionMovements) { LOGGER.info("moveStuckPartitions: Skip moving stuck partition " + tp + " from " + itph.getInstanceName() + " because moving reaches upper bound of " + _maxStuckPartitionMovements); continue; } else if (!moveHistory.isEmpty() && moveHistory.get(moveHistory.size() - 1) + _movePartitionAfterStuckMillis > now) { LOGGER.info("moveStuckPartitions: Skip moving stuck partition " + tp + " from " + itph.getInstanceName() + " because it was moved recently"); continue; LOGGER.info("moveStuckPartitions: Trying to move stuck partition " + tp + " from " + itph.getInstanceName()); moveHistory.add(now); itph.removeTopicPartition(tp);
while (instances.size() > _maxWorkingInstances && iter.hasNext()) { InstanceTopicPartitionHolder itph = iter.next(); LOGGER.info("Move workload from instance " + itph.getInstanceName() + " to become idle: " + itph.getServingTopicPartitionSet()); tpiNeedsToBeAssigned.addAll(itph.getServingTopicPartitionSet());
@Override public int compare(InstanceTopicPartitionHolder o1, InstanceTopicPartitionHolder o2) { TopicWorkload workload1 = (o1 == null) ? new TopicWorkload(0, 0) : o1.totalWorkload(infoRetriever, weighter); TopicWorkload workload2 = (o2 == null) ? new TopicWorkload(0, 0) : o2.totalWorkload(infoRetriever, weighter); int cmp = workload1.compareTotal(workload2); if (cmp != 0) { return cmp; } // if workload is the same, compare them based on the number of partitions int size1 = (o1 == null) ? -1 : o1.getNumServingTopicPartitions(); int size2 = (o2 == null) ? -1 : o2.getNumServingTopicPartitions(); if (size1 != size2) { return size1 - size2; } else { return o1.getInstanceName().compareTo(o2.getInstanceName()); } } };
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); if (liveInstance != null) { customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { synchronized (instanceToNumServingTopicPartitionMap) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
instancesForNewPartitions.get(i % instancesForNewPartitions.size()); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); LOGGER.info("Assign new partition " + topicName + ":" + i + " to instance " + liveInstance.getInstanceName());
while (iter.hasNext()) { InstanceTopicPartitionHolder instance = iter.next(); String name = instance.getInstanceName(); if (instanceName == null || instanceName.equals(name)) { if (!instanceMapJson.containsKey(name)) {
instancesPartitionsCount.put(instance.getInstanceName(), partitionCount); if (instancesPartitionsCount.get(instance.getInstanceName()) == 1) { for (TopicPartition tp : instance.getServingTopicPartitionSet()) { if (weighter.partitionWeight(tp) > 0) { InstanceTopicPartitionHolder highest = orderedInstances.pollLast(); processedInstances.add(highest); if (instancesPartitionsCount.get(highest.getInstanceName()) <= 1) {