public static IdealState addNewRealtimeSegmentToIdealState(String segmentId, IdealState state, String instanceName) { state.setPartitionState(segmentId, instanceName, ONLINE); state.setNumPartitions(state.getNumPartitions() + 1); return state; }
.assertEquals(_helixAdmin.getResourceIdealState(_helixClusterName, tableName + "_OFFLINE").getNumPartitions(), i); _helixResourceManager.addNewSegment(SegmentMetadataMockUtils.mockSegmentMetadata(tableName), "downloadUrl"); Assert .assertEquals(_helixAdmin.getResourceIdealState(_helixClusterName, tableName + "_OFFLINE").getNumPartitions(), i + 1);
Assert.assertEquals(_helixAdmin.getResourceIdealState(_helixClusterName, OFFLINE_TABLE_NAME).getNumPartitions(), i + 1);
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) { Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>(); HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) { IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic); int numPartitions = is.getNumPartitions(); for (int partition = 0; partition < numPartitions; ++partition) { if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) { TopicPartition tpi = new TopicPartition(topic, partition); unassignedPartitions.add(tpi); } } } return unassignedPartitions; }
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
} else { int numPartitionsInMirrorMaker = _helixMirrorMakerManager.getIdealStateForTopic(topic).getNumPartitions(); if (numPartitionsInMirrorMaker != tp.getPartition()) { int mismatchedPartitions = Math.abs(numPartitionsInMirrorMaker - tp.getPartition());
ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName); numTopicPartitions += idealStateForTopic.getNumPartitions(); if (idealStateForTopic.getNumPartitions() != externalViewForTopic.getPartitionSet() .size()) { numErrorTopics++;
public static IdealState expandCustomRebalanceModeIdealStateFor(IdealState oldIdealState, String topicName, int newNumTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(newNumTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(newNumTopicPartitions); int numOldPartitions = oldIdealState.getNumPartitions(); for (int i = 0; i < numOldPartitions; ++i) { String partitionName = Integer.toString(i); try { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } catch (Exception e) { // No worker added into the cluster. } } for (int i = numOldPartitions; i < newNumTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } return customModeIdealStateBuilder.build(); }
public void refreshWorkloads() throws IOException { long current = System.currentTimeMillis(); if (_lastRefreshTimeMillis + _minRefreshIntervalMillis > current) { LOGGER.info("Too soon to refresh workload, skip"); return; } LOGGER.info("Refreshing workload for source " + _srcKafkaCluster); _lastRefreshTimeMillis = current; List<String> topics = _helixMirrorMakerManager.getTopicLists(); Map<String, Integer> topicsPartitions = new HashMap<>(); for (String topic : topics) { IdealState idealState = _helixMirrorMakerManager.getIdealStateForTopic(topic); if (idealState != null) { int partitions = idealState.getNumPartitions(); if (partitions > 0) { topicsPartitions.put(topic, partitions); } } } retrieveWorkload(current, DEFAULT_WORKLOAD_WINDOW_MILLIS, topicsPartitions); }
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) { Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>(); HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) { IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic); int numPartitions = is.getNumPartitions(); for (int partition = 0; partition < numPartitions; ++partition) { if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) { TopicPartition tpi = new TopicPartition(topic, partition); unassignedPartitions.add(tpi); } } } return unassignedPartitions; }
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) { Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>(); HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) { IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic); int numPartitions = is.getNumPartitions(); for (int partition = 0; partition < numPartitions; ++partition) { if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) { TopicPartition tpi = new TopicPartition(topic, partition); unassignedPartitions.add(tpi); } } } return unassignedPartitions; }
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
public static IdealState resetCustomIdealStateFor(IdealState oldIdealState, String topicName, String oldPartition, String newPartition, String newInstanceName) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions).setNumReplica(1) .setMaxPartitionsPerNode(oldNumPartitions); for (String partitionName : oldIdealState.getPartitionSet()) { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); String instanceToUse = partitionName.equals(oldPartition) ? newInstanceName : instanceName; String partitionToUse = partitionName.equals(oldPartition) ? newPartition : oldPartition; customModeIdealStateBuilder.assignInstanceAndState(partitionToUse, instanceToUse, "ONLINE"); } return customModeIdealStateBuilder.build(); }
public static IdealState resetCustomIdealStateFor(IdealState oldIdealState, String topicName, List<String> instanceToReplace, List<String> availableInstances, int maxNumReplica) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions).setNumReplica(maxNumReplica) .setMaxPartitionsPerNode(oldNumPartitions); for (String partitionName : oldIdealState.getPartitionSet()) { for (String instanceName : oldIdealState.getInstanceStateMap(partitionName).keySet()) { String instanceToUse = instanceToReplace.contains(instanceName) ? availableInstances.get(0) : instanceName; customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceToUse, "ONLINE"); if (instanceToReplace.contains(instanceName)) { availableInstances.remove(0); LOGGER.info("replaceing: route: {}@{}, old {}, new {}", topicName, partitionName, instanceName, instanceToUse); } } } return customModeIdealStateBuilder.build(); }
public static IdealState resetCustomIdealStateFor(IdealState oldIdealState, String topicName, String partitionToReplace, String newInstanceName) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions).setNumReplica(1) .setMaxPartitionsPerNode(oldNumPartitions); for (String partitionName : oldIdealState.getPartitionSet()) { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); String instanceToUse = partitionName.equals(partitionToReplace) ? newInstanceName : instanceName; customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceToUse, "ONLINE"); } return customModeIdealStateBuilder.build(); }
public static IdealState shrinkCustomIdealStateFor(IdealState oldIdealState, String topicName, String partitionToDelete) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions - 1).setNumReplica(1) .setMaxPartitionsPerNode(oldNumPartitions - 1); for (String partitionName : oldIdealState.getPartitionSet()) { if (!partitionName.equals(partitionToDelete)) { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } } return customModeIdealStateBuilder.build(); }
public static IdealState expandCustomIdealStateFor(IdealState oldIdealState, String topicName, String newPartition, InstanceTopicPartitionHolder instance) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions + 1).setNumReplica(1) .setMaxPartitionsPerNode(oldNumPartitions + 1); for (String partitionName : oldIdealState.getPartitionSet()) { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } customModeIdealStateBuilder.assignInstanceAndState(newPartition, instance.getInstanceName(), "ONLINE"); return customModeIdealStateBuilder.build(); }
public static IdealState shrinkInstanceCustomIdealStateFor(IdealState oldIdealState, String topicName, String partition, List<String> instancesToRemove, int maxNumReplica) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions).setNumReplica(maxNumReplica) .setMaxPartitionsPerNode(oldNumPartitions); for (String partitionName : oldIdealState.getPartitionSet()) { if (partitionName.equals(partition)) { for (String instanceName : oldIdealState.getInstanceStateMap(partitionName).keySet()) { if (!instancesToRemove.contains(instanceName)) { customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } } } else { for (String instanceName : oldIdealState.getInstanceStateMap(partitionName).keySet()) { customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } } } return customModeIdealStateBuilder.build(); }
public static IdealState expandInstanceCustomIdealStateFor(IdealState oldIdealState, String topicName, String newPartition, List<String> instances, int maxNumReplica) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions).setNumReplica(maxNumReplica) .setMaxPartitionsPerNode(oldNumPartitions); for (String partitionName : oldIdealState.getPartitionSet()) { for (String instanceName : oldIdealState.getInstanceStateMap(partitionName).keySet()) { customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } if (partitionName.equals(newPartition)) { for (String newInstanceName : instances) { customModeIdealStateBuilder.assignInstanceAndState(partitionName, newInstanceName, "ONLINE"); } } } return customModeIdealStateBuilder.build(); }
public static IdealState expandCustomIdealStateFor(IdealState oldIdealState, String topicName, String newPartition, List<String> instances, int maxNumReplica) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions + 1).setNumReplica(maxNumReplica) .setMaxPartitionsPerNode(oldNumPartitions + 1); for (String partitionName : oldIdealState.getPartitionSet()) { for (String instanceName : oldIdealState.getInstanceStateMap(partitionName).keySet()) { customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } } for (String instance : instances) { customModeIdealStateBuilder.assignInstanceAndState(newPartition, instance, "ONLINE"); } return customModeIdealStateBuilder.build(); }