private static Set<InstanceTopicPartitionHolder> balanceAssignment( TreeSet<InstanceTopicPartitionHolder> orderedSet) { while (!isAssignmentBalanced(orderedSet)) { InstanceTopicPartitionHolder lowestInstance = orderedSet.pollFirst(); InstanceTopicPartitionHolder highestInstance = orderedSet.pollLast(); TopicPartition tpi = highestInstance.getServingTopicPartitionSet().iterator().next(); highestInstance.removeTopicPartition(tpi); lowestInstance.addTopicPartition(tpi); orderedSet.add(lowestInstance); orderedSet.add(highestInstance); } return orderedSet; }
public static IdealState expandCustomRebalanceModeIdealStateFor(IdealState oldIdealState, String topicName, int newNumTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(newNumTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(newNumTopicPartitions); int numOldPartitions = oldIdealState.getNumPartitions(); for (int i = 0; i < numOldPartitions; ++i) { String partitionName = Integer.toString(i); try { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } catch (Exception e) { // No worker added into the cluster. } } for (int i = numOldPartitions; i < newNumTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } return customModeIdealStateBuilder.build(); }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); if (liveInstance != null) { customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { synchronized (instanceToNumServingTopicPartitionMap) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
lowestInstances.get(j % lowestInstances.size()).addTopicPartition(sameTopic.get(j));
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); if (liveInstance != null) { customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { synchronized (instanceToNumServingTopicPartitionMap) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); LOGGER.info("Assign new partition " + topicName + ":" + i + " to instance " + liveInstance.getInstanceName());