public static List<String> getPreferenceList(Partition partition, IdealState idealState, Set<String> eligibleInstances) { List<String> listField = idealState.getPreferenceList(partition.getPartitionName()); if (listField != null && listField.size() == 1 && IdealState.IdealStateConstants.ANY_LIVEINSTANCE.toString().equals(listField.get(0))) { List<String> prefList = new ArrayList<String>(eligibleInstances); Collections.sort(prefList); return prefList; } else { return listField; } }
public static List<String> getPreferenceList(Partition partition, IdealState idealState, Set<String> eligibleInstances) { List<String> listField = idealState.getPreferenceList(partition.getPartitionName()); if (listField != null && listField.size() == 1 && IdealState.IdealStateConstants.ANY_LIVEINSTANCE.toString().equals(listField.get(0))) { List<String> prefList = new ArrayList<String>(eligibleInstances); Collections.sort(prefList); return prefList; } else { return listField; } }
/** * Check if the provided list matches the currently persisted preference list * @param admin * @param clusterName * @param resourceName * @param partitionName * @param expectPreferenceList * @return */ private boolean preferenceListIsCorrect(HelixAdmin admin, String clusterName, String resourceName, String partitionName, List<String> expectPreferenceList) { IdealState idealState = admin.getResourceIdealState(clusterName, resourceName); List<String> preferenceList = idealState.getPreferenceList(partitionName); return expectPreferenceList.equals(preferenceList); }
List<String> preferenceList = idealState.getPreferenceList(partition.getPartitionName()); if (preferenceList == null) { preferenceList = Collections.emptyList();
List<String> preferenceList = idealState.getPreferenceList(partition.getPartitionName()); if (preferenceList == null) { preferenceList = Collections.emptyList();
private void verifyUserDefinedPreferenceLists(String db, Map<String, List<String>> userDefinedPreferenceLists, List<String> userDefinedPartitions) throws InterruptedException { IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); for (String p : userDefinedPreferenceLists.keySet()) { List<String> userDefined = userDefinedPreferenceLists.get(p); List<String> preferenceListInIs = is.getPreferenceList(p); if (userDefinedPartitions.contains(p)) { Assert.assertTrue(userDefined.equals(preferenceListInIs)); } else { if (userDefined.equals(preferenceListInIs)) { Assert.fail("Something is not good!"); } Assert.assertFalse(userDefined.equals(preferenceListInIs), String .format("Partition %s, List in Is: %s, List as defined in config: %s", p, preferenceListInIs, userDefined)); } } }
private void verifyNonUserDefinedAssignment(String db, IdealState originIS, Set<String> nonUserDefinedPartitions) throws InterruptedException { IdealState newIS = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); Assert.assertEquals(originIS.getPartitionSet(), newIS.getPartitionSet()); for (String p : newIS.getPartitionSet()) { if (nonUserDefinedPartitions.contains(p)) { // for non user defined partition, mapping should keep the same Assert.assertEquals(newIS.getPreferenceList(p), originIS.getPreferenceList(p)); } } }
Set<String> partitionSet = getPartitionSet(); for (String partition : partitionSet) { List<String> preferenceList = getPreferenceList(partition); if (preferenceList == null || preferenceList.size() != replica) { logger
Set<String> partitionSet = getPartitionSet(); for (String partition : partitionSet) { List<String> preferenceList = getPreferenceList(partition); if (preferenceList == null || preferenceList.size() != replica) { logger
private void verifySemiAutoMasterSlaveAssignment(IdealState idealState) { for (String partition : idealState.getPartitionSet()) { Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition); List<String> preferenceList = idealState.getPreferenceList(partition); int numMaster = 0; for (String ins : preferenceList) { Assert.assertTrue(instanceStateMap.containsKey(ins), String.format("Instance %s from preference list not in the map", ins)); String state = instanceStateMap.get(ins); Assert.assertTrue(state.equals(MasterSlaveSMD.States.MASTER.name()) || state .equals(MasterSlaveSMD.States.SLAVE.name()), "Actual State" + state); if (state.equals(MasterSlaveSMD.States.MASTER.name())) { numMaster++; } } Assert.assertEquals(numMaster, 1); } }
@Override public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState, CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) { testRebalancerInvoked = true; List<String> liveNodes = Lists.newArrayList(clusterData.getLiveInstances().keySet()); int i = 0; for (String partition : currentIdealState.getPartitionSet()) { int index = i++ % liveNodes.size(); String instance = liveNodes.get(index); currentIdealState.getPreferenceList(partition).clear(); currentIdealState.getPreferenceList(partition).add(instance); currentIdealState.getInstanceStateMap(partition).clear(); currentIdealState.getInstanceStateMap(partition).put(instance, "MASTER"); } currentIdealState.setReplicas("1"); return currentIdealState; } }
private void verifyAssignmentInIdealStateWithPersistEnabled(IdealState idealState, Set<String> excludedInstances) { for (String partition : idealState.getPartitionSet()) { Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition); Assert.assertNotNull(instanceStateMap); Assert.assertFalse(instanceStateMap.isEmpty()); Set<String> instancesInMap = instanceStateMap.keySet(); if (idealState.getRebalanceMode() == RebalanceMode.SEMI_AUTO) { Set<String> instanceInList = idealState.getInstanceSet(partition); Assert.assertTrue(instanceInList.containsAll(instancesInMap)); } if(idealState.getRebalanceMode() == RebalanceMode.FULL_AUTO) { // preference list should be persisted in IS. List<String> instanceList = idealState.getPreferenceList(partition); Assert.assertNotNull(instanceList); Assert.assertFalse(instanceList.isEmpty()); for (String ins : excludedInstances) { Assert.assertFalse(instanceList.contains(ins)); } } for (String ins : excludedInstances) { Assert.assertFalse(instancesInMap.contains(ins)); } } }
private void verifyAssignmentInIdealStateWithPersistDisabled(IdealState idealState, Set<String> excludedInstances) { boolean mapFieldEmpty = true; boolean assignmentNotChanged = false; for (String partition : idealState.getPartitionSet()) { Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition); if (instanceStateMap == null || instanceStateMap.isEmpty()) { continue; } mapFieldEmpty = false; Set<String> instancesInMap = instanceStateMap.keySet(); for (String ins : excludedInstances) { if(instancesInMap.contains(ins)) { // if at least one excluded instance is included, it means assignment was not updated. assignmentNotChanged = true; } if(idealState.getRebalanceMode() == RebalanceMode.FULL_AUTO) { List<String> instanceList = idealState.getPreferenceList(partition); if (instanceList.contains(ins)) { assignmentNotChanged = true; } } } } Assert.assertTrue((mapFieldEmpty || assignmentNotChanged), "BestPossible assignment was updated."); } }
if (idealState.getPreferenceList(partition) == null || idealState.getPreferenceList(partition).isEmpty()) { return false;
if (idealState.getPreferenceList(partition) == null || idealState.getPreferenceList(partition).isEmpty()) { return false;
private void verify(String dbName) { IdealState idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, dbName); HelixDataAccessor accessor = _manager.getHelixDataAccessor(); ExternalView ev = accessor.getProperty(accessor.keyBuilder().externalView(dbName)); for (String p : idealState.getPartitionSet()) { String idealMaster = idealState.getPreferenceList(p).get(0); Assert.assertTrue(ev.getStateMap(p).get(idealMaster).equals("ERROR")); TimeOutStateModel model = _factories.get(idealMaster).getStateModel(dbName, p); Assert.assertEquals(model._errorCallcount, 1); Assert.assertEquals(model._error.getCode(), ErrorCode.TIMEOUT); } }
private void setParticipants(String dbName) throws InterruptedException { _factories = new HashMap<>(); IdealState idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, dbName); for (int i = 0; i < NODE_NR; i++) { if (_participants[i] != null) { _participants[i].syncStop(); } Thread.sleep(1000); String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i); SleepStateModelFactory factory = new SleepStateModelFactory(1000); _factories.put(instanceName, factory); for (String p : idealState.getPartitionSet()) { if (idealState.getPreferenceList(p).get(0).equals(instanceName)) { factory.addPartition(p); } } _participants[i] = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instanceName); _participants[i].getStateMachineEngine().registerStateModelFactory("MasterSlave", factory); _participants[i].syncStart(); } } }
factories.put(instanceName, factory); for (String p : idealState.getPartitionSet()) { if (idealState.getPreferenceList(p).get(0).equals(instanceName)) { factory.addPartition(p); ExternalView ev = accessor.getProperty(kb.externalView(TEST_DB)); for (String p : idealState.getPartitionSet()) { String idealMaster = idealState.getPreferenceList(p).get(0); Assert.assertTrue(ev.getStateMap(p).get(idealMaster).equals("ERROR"));
@Test public void testCustomizedIdealStateRebalancer() throws InterruptedException { _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "MasterSlave"); _gSetupTool.addResourceProperty(CLUSTER_NAME, db2, IdealStateProperty.REBALANCER_CLASS_NAME.toString(), TestCustomizedIdealStateRebalancer.TestRebalancer.class.getName()); _gSetupTool.addResourceProperty(CLUSTER_NAME, db2, IdealStateProperty.REBALANCE_MODE.toString(), RebalanceMode.USER_DEFINED.toString()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, 3); boolean result = ClusterStateVerifier.verifyByZkCallback(new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, db2)); Assert.assertTrue(result); Thread.sleep(1000); HelixDataAccessor accessor = new ZKHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); ExternalView ev = accessor.getProperty(keyBuilder.externalView(db2)); Assert.assertEquals(ev.getPartitionSet().size(), 60); for (String partition : ev.getPartitionSet()) { Assert.assertEquals(ev.getStateMap(partition).size(), 1); } IdealState is = accessor.getProperty(keyBuilder.idealStates(db2)); for (String partition : is.getPartitionSet()) { Assert.assertEquals(is.getPreferenceList(partition).size(), 0); Assert.assertEquals(is.getInstanceStateMap(partition).size(), 0); } Assert.assertTrue(testRebalancerCreated); Assert.assertTrue(testRebalancerInvoked); }
public void verifyReplication(HelixZkClient zkClient, String clusterName, String resource, int repl) { ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient)); Builder keyBuilder = accessor.keyBuilder(); IdealState idealState = accessor.getProperty(keyBuilder.idealStates(resource)); for (String partitionName : idealState.getPartitionSet()) { if (idealState.getRebalanceMode() == IdealState.RebalanceMode.SEMI_AUTO) { AssertJUnit.assertEquals(repl, idealState.getPreferenceList(partitionName).size()); } else if (idealState.getRebalanceMode() == IdealState.RebalanceMode.CUSTOMIZED) { AssertJUnit.assertEquals(repl, idealState.getInstanceStateMap(partitionName).size()); } } }