@Override public IdealState apply(IdealState idealState) { Set<String> partitions = idealState.getPartitionSet(); if (partitions.contains(segmentName)) { LOGGER.warn("Segment already exists in the ideal state for segment: {} of table: {}, do not update", segmentName, tableNameWithType); } else { if (assignedInstances.isEmpty()) { LOGGER.warn("No instance assigned for segment: {} of table: {}", segmentName, tableNameWithType); } else { int numPartitions = partitions.size() + 1; for (String instance : assignedInstances) { idealState.setPartitionState(segmentName, instance, ONLINE); } idealState.setNumPartitions(numPartitions); } } return idealState; } };
public static IdealState addNewRealtimeSegmentToIdealState(String segmentId, IdealState state, String instanceName) { state.setPartitionState(segmentId, instanceName, ONLINE); state.setNumPartitions(state.getNumPartitions() + 1); return state; }
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<String, CustomModeISBuilder>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<String, IdealState>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
@Override public void addClusterToGrandCluster(String clusterName, String grandCluster) { logger.info("Add cluster {} to grand cluster {}.", clusterName, grandCluster); if (!ZKUtil.isClusterSetup(grandCluster, _zkClient)) { throw new HelixException("Grand cluster " + grandCluster + " is not setup yet"); } if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("Cluster " + clusterName + " is not setup yet"); } IdealState idealState = new IdealState(clusterName); idealState.setNumPartitions(1); idealState.setStateModelDefRef("LeaderStandby"); List<String> controllers = getInstancesInCluster(grandCluster); if (controllers.size() == 0) { throw new HelixException("Grand cluster " + grandCluster + " has no instances"); } idealState.setReplicas(Integer.toString(controllers.size())); Collections.shuffle(controllers); idealState.getRecord().setListField(clusterName, controllers); idealState.setPartitionState(clusterName, controllers.get(0), "LEADER"); for (int i = 1; i < controllers.size(); i++) { idealState.setPartitionState(clusterName, controllers.get(i), "STANDBY"); } ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(grandCluster, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.idealStates(idealState.getResourceName()), idealState); }
@Override public void addClusterToGrandCluster(String clusterName, String grandCluster) { logger.info("Add cluster {} to grand cluster {}.", clusterName, grandCluster); if (!ZKUtil.isClusterSetup(grandCluster, _zkClient)) { throw new HelixException("Grand cluster " + grandCluster + " is not setup yet"); } if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("Cluster " + clusterName + " is not setup yet"); } IdealState idealState = new IdealState(clusterName); idealState.setNumPartitions(1); idealState.setStateModelDefRef("LeaderStandby"); List<String> controllers = getInstancesInCluster(grandCluster); if (controllers.size() == 0) { throw new HelixException("Grand cluster " + grandCluster + " has no instances"); } idealState.setReplicas(Integer.toString(controllers.size())); Collections.shuffle(controllers); idealState.getRecord().setListField(clusterName, controllers); idealState.setPartitionState(clusterName, controllers.get(0), "LEADER"); for (int i = 1; i < controllers.size(); i++) { idealState.setPartitionState(clusterName, controllers.get(i), "STANDBY"); } ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(grandCluster, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.idealStates(idealState.getResourceName()), idealState); }
@Test(dataProvider = "IdealStateInput") public void testIdealStateAssignment(String clusterName, List<String> instances, List<String> partitions, String numReplicas, String stateModeDef, String strategyName, Map<String, Map<String, String>> expectedMapping) throws IllegalAccessException, InstantiationException, ClassNotFoundException { ClusterConfig clusterConfig = new ClusterConfig(clusterName); List<InstanceConfig> instanceConfigs = new ArrayList<>(); for (String instance : instances) { instanceConfigs.add(new InstanceConfig(instance)); } IdealState idealState = new IdealState("TestResource"); idealState.setStateModelDefRef(stateModeDef); idealState.setNumPartitions(partitions.size()); idealState.setReplicas(numReplicas); Map<String, Map<String, String>> idealStateMapping = HelixUtil .getIdealAssignmentForFullAuto(clusterConfig, instanceConfigs, instances, idealState, partitions, strategyName); Assert.assertEquals(idealStateMapping, expectedMapping); }
@Test public void testReplicas() { IdealState idealState = new IdealState("test-db"); idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO); idealState.setNumPartitions(4); idealState.setStateModelDefRef("MasterSlave"); idealState.setReplicas("" + 2); List<String> preferenceList = new ArrayList<String>(); preferenceList.add("node_0"); idealState.getRecord().setListField("test-db_0", preferenceList); Assert.assertFalse(idealState.isValid(), "should fail since replicas not equals to preference-list size"); preferenceList.add("node_1"); idealState.getRecord().setListField("test-db_0", preferenceList); Assert.assertTrue(idealState.isValid(), "should pass since replicas equals to preference-list size"); }
idealState.setStateModelDefRef("MasterSlave"); idealState.setIdealStateMode(mode.toString()); idealState.setNumPartitions(partitions); idealStates.add(idealState);
private void updateIdealState() { HelixDataAccessor accessor = _manager.getHelixDataAccessor(); Builder keyBuilder = accessor.keyBuilder(); List<String> idealStates = accessor.getChildNames(keyBuilder.idealStates()); IdealState idealState = accessor.getProperty(keyBuilder.idealStates(idealStates.get(0))); idealState.setNumPartitions((int)(System.currentTimeMillis()%50L)); accessor.setProperty(keyBuilder.idealStates(idealState.getId()), idealState); } }
protected List<IdealState> setupIdealState(String clusterName, int[] nodes, String[] resources, int partitions, int replicas) { ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); List<IdealState> idealStates = new ArrayList<IdealState>(); List<String> instances = new ArrayList<String>(); for (int i : nodes) { instances.add("localhost_" + i); } for (String resourceName : resources) { IdealState idealState = new IdealState(resourceName); for (int p = 0; p < partitions; p++) { List<String> value = new ArrayList<String>(); for (int r = 0; r < replicas; r++) { int n = nodes[(p + r) % nodes.length]; value.add("localhost_" + n); } idealState.getRecord().setListField(resourceName + "_" + p, value); } idealState.setReplicas(Integer.toString(replicas)); idealState.setStateModelDefRef("MasterSlave"); idealState.setRebalanceMode(IdealState.RebalanceMode.SEMI_AUTO); idealState.setNumPartitions(partitions); idealStates.add(idealState); // System.out.println(idealState); accessor.setProperty(keyBuilder.idealStates(resourceName), idealState); } return idealStates; }
private void createIS(HelixDataAccessor accessor, String resourceId, String stateModelDefRef, RebalanceMode rebalanceMode) { IdealState idealState = new IdealState(resourceId); idealState.setRebalanceMode(rebalanceMode); idealState.setStateModelDefRef(stateModelDefRef); idealState.setNumPartitions(1); idealState.setReplicas("1"); idealState.getRecord().setListField(resourceId + "_0", ImmutableList.of(PARTICIPANT)); idealState.getRecord().setMapField(resourceId + "_0", ImmutableMap.of(PARTICIPANT, STATE)); accessor.setProperty(accessor.keyBuilder().idealStates(resourceId), idealState); }
/** * Create an IdealState for a resource that belongs to a resource group We use * "resourceGroupName$resourceInstanceTag" as the IdealState znode name to differetiate different * resources from the same resourceGroup. */ public IdealState createIdealStateForResourceGroup(String resourceGroupName, String resourceTag, int numPartition, int replica, String rebalanceMode, String stateModelDefName) { String idealStateId = genIdealStateNameWithResourceTag(resourceGroupName, resourceTag); IdealState idealState = new IdealState(idealStateId); idealState.setNumPartitions(numPartition); idealState.setStateModelDefRef(stateModelDefName); IdealState.RebalanceMode mode = idealState.rebalanceModeFromString(rebalanceMode, IdealState.RebalanceMode.SEMI_AUTO); idealState.setRebalanceMode(mode); idealState.setReplicas("" + replica); idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY); idealState.setResourceGroupName(resourceGroupName); idealState.setInstanceGroupTag(resourceTag); idealState.enableGroupRouting(true); return idealState; }
/** * Create an IdealState for a resource that belongs to a resource group We use * "resourceGroupName$resourceInstanceTag" as the IdealState znode name to differetiate different * resources from the same resourceGroup. */ public IdealState createIdealStateForResourceGroup(String resourceGroupName, String resourceTag, int numPartition, int replica, String rebalanceMode, String stateModelDefName) { String idealStateId = genIdealStateNameWithResourceTag(resourceGroupName, resourceTag); IdealState idealState = new IdealState(idealStateId); idealState.setNumPartitions(numPartition); idealState.setStateModelDefRef(stateModelDefName); IdealState.RebalanceMode mode = idealState.rebalanceModeFromString(rebalanceMode, IdealState.RebalanceMode.SEMI_AUTO); idealState.setRebalanceMode(mode); idealState.setReplicas("" + replica); idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY); idealState.setResourceGroupName(resourceGroupName); idealState.setInstanceGroupTag(resourceTag); idealState.enableGroupRouting(true); return idealState; }
idealState.setNumPartitions(partitions); idealStates.add(idealState); idealState.setReplicas(String.valueOf(replicas));
private void updateConfigs() throws InterruptedException { final Random r = new Random(System.currentTimeMillis()); // test change content HelixDataAccessor accessor = _manager.getHelixDataAccessor(); Builder keyBuilder = accessor.keyBuilder(); final List<String> instances = accessor.getChildNames(keyBuilder.instanceConfigs()); for (String instance : instances) { InstanceConfig value = accessor.getProperty(keyBuilder.instanceConfig(instance)); value._record.setLongField("TimeStamp", System.currentTimeMillis()); accessor.setProperty(keyBuilder.instanceConfig(instance), value); Thread.sleep(50); } final List<String> resources = accessor.getChildNames(keyBuilder.idealStates()); for (String resource : resources) { IdealState idealState = accessor.getProperty(keyBuilder.idealStates(resource)); idealState.setNumPartitions(r.nextInt(100)); accessor.setProperty(keyBuilder.idealStates(idealState.getId()), idealState); Thread.sleep(20); // wait zk callback } } }
@Override public void addResource(String clusterName, String resourceName, int partitions, String stateModelRef, String rebalancerMode, String rebalanceStrategy, int bucketSize, int maxPartitionsPerInstance) { if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("cluster " + clusterName + " is not setup yet"); } IdealState idealState = new IdealState(resourceName); idealState.setNumPartitions(partitions); idealState.setStateModelDefRef(stateModelRef); RebalanceMode mode = idealState.rebalanceModeFromString(rebalancerMode, RebalanceMode.SEMI_AUTO); idealState.setRebalanceMode(mode); idealState.setRebalanceStrategy(rebalanceStrategy); idealState.setReplicas("" + 0); idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY); if (maxPartitionsPerInstance > 0 && maxPartitionsPerInstance < Integer.MAX_VALUE) { idealState.setMaxPartitionsPerInstance(maxPartitionsPerInstance); } if (bucketSize > 0) { idealState.setBucketSize(bucketSize); } addResource(clusterName, resourceName, idealState); }
@Override public void addResource(String clusterName, String resourceName, int partitions, String stateModelRef, String rebalancerMode, String rebalanceStrategy, int bucketSize, int maxPartitionsPerInstance) { if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("cluster " + clusterName + " is not setup yet"); } IdealState idealState = new IdealState(resourceName); idealState.setNumPartitions(partitions); idealState.setStateModelDefRef(stateModelRef); RebalanceMode mode = idealState.rebalanceModeFromString(rebalancerMode, RebalanceMode.SEMI_AUTO); idealState.setRebalanceMode(mode); idealState.setRebalanceStrategy(rebalanceStrategy); idealState.setReplicas("" + 0); idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY); if (maxPartitionsPerInstance > 0 && maxPartitionsPerInstance < Integer.MAX_VALUE) { idealState.setMaxPartitionsPerInstance(maxPartitionsPerInstance); } if (bucketSize > 0) { idealState.setBucketSize(bucketSize); } addResource(clusterName, resourceName, idealState); }
IdealState is = new IdealState(resourceName); is.setStateModelDefRef("OnlineOffline"); is.setNumPartitions(2); is.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO); is.setReplicas("1");