_gSetupTool.addResourceToCluster(CLUSTER_NAME, db1, NUM_PARTITIONS, "MasterSlave", IdealState.RebalanceMode.FULL_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db1, NUM_REPLICAS); _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, NUM_PARTITIONS, "MasterSlave", IdealState.RebalanceMode.FULL_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, NUM_REPLICAS);
@Test (dependsOnMethods = "testMaintenanceModeAddNewInstance") public void testMaintenanceModeAddNewResource() throws InterruptedException { _gSetupTool.getClusterManagementTool() .addResource(CLUSTER_NAME, newResourceAddedDuringMaintenanceMode, 7, "MasterSlave", IdealState.RebalanceMode.FULL_AUTO.name()); _gSetupTool.getClusterManagementTool() .rebalance(CLUSTER_NAME, newResourceAddedDuringMaintenanceMode, 3); Assert.assertTrue(_clusterVerifier.verifyByPolling()); ExternalView externalView = _gSetupTool.getClusterManagementTool() .getResourceExternalView(CLUSTER_NAME, newResourceAddedDuringMaintenanceMode); Assert.assertNull(externalView); }
_gSetupTool.addResourceToCluster(CLUSTER_NAME, db1, NUM_PARTITIONS, "MasterSlave", IdealState.RebalanceMode.FULL_AUTO.name()); long startTime = System.currentTimeMillis(); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db1, NUM_REPLICAS); _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, NUM_PARTITIONS, "MasterSlave", IdealState.RebalanceMode.FULL_AUTO.name()); startTime = System.currentTimeMillis(); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, NUM_REPLICAS);
IdealState.RebalanceMode.SEMI_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, "db-semi", _replica); IdealState.RebalanceMode.CUSTOMIZED.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, "db-customized", _replica); IdealState.RebalanceMode.FULL_AUTO.name(), RebalanceStrategy.DEFAULT_REBALANCE_STRATEGY); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, "db-fa", _replica);
BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO.name(), CrushRebalanceStrategy.class.getName()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, testDb, replicas);
IdealState.RebalanceMode.SEMI_AUTO.name());
String dbName = "TestDB_" + i; _gSetupTool.addResourceToCluster(CLUSTER_NAME, dbName, _PARTITIONS, STATE_MODEL, IdealState.RebalanceMode.CUSTOMIZED.name()); _gSetupTool.rebalanceResource(CLUSTER_NAME, dbName, 3);
BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.SEMI_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, testDb, 3);
@Test public void testParticipantUnavailable() throws Exception { _gSetupTool.addResourceToCluster(CLUSTER_NAME, testDb, 5, BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, testDb, 3); ZkHelixClusterVerifier verifier = BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, testDb, 3); Assert.assertTrue(verifier.verifyByPolling());
MASTER_SLAVE_STATE_MODEL, IdealState.RebalanceMode.FULL_AUTO.name());
String resourceName = "TEST_RESOURCE"; _gSetupTool.getClusterManagementTool().addResource(CLUSTER_NAME, resourceName, 1, "MasterSlave", IdealState.RebalanceMode.CUSTOMIZED.name()); String partitionName = "P_0"; ClusterEvent event = new ClusterEvent(CLUSTER_NAME, ClusterEventType.Unknown, "ID");
@Test(dataProvider = "rebalanceModes", dependsOnMethods = {"testDisablePersist"}) public void testEnablePersist(RebalanceMode rebalanceMode) throws Exception { String testDb = "TestDB1-" + rebalanceMode.name(); enablePersistBestPossibleAssignment(_gZkClient, CLUSTER_NAME, true); BuiltInStateModelDefinitions.LeaderStandby.name(), rebalanceMode.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, testDb, 3);
@Test(dataProvider = "rebalanceModes") public void testDisablePersist(RebalanceMode rebalanceMode) throws Exception { String testDb = "TestDB2-" + rebalanceMode.name(); _gSetupTool.addResourceToCluster(CLUSTER_NAME, testDb, 5, BuiltInStateModelDefinitions.LeaderStandby.name(), rebalanceMode.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, testDb, 3); BestPossibleExternalViewVerifier.Builder verifierBuilder = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR) .setResources(new HashSet<String>(Collections.singleton(testDb))); Assert.assertTrue(verifierBuilder.build().verifyByPolling()); // kill 1 node _participants[0].syncStop(); Set<String> liveInstances = new HashSet<String>(_instanceNames); liveInstances.remove(_participants[0].getInstanceName()); verifierBuilder.setExpectLiveInstances(liveInstances); Assert.assertTrue(verifierBuilder.build().verifyByPolling()); IdealState idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, testDb); Set<String> excludedInstances = new HashSet<String>(); excludedInstances.add(_participants[0].getInstanceName()); verifyAssignmentInIdealStateWithPersistDisabled(idealState, excludedInstances); // clean up _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME, testDb); _participants[0] = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, _participants[0].getInstanceName()); _participants[0].syncStart(); }
@Test public void testANYtypeThrottle() throws InterruptedException { // start some participants for (int i = 0; i < NODE_NR - 3; i++) { _participants[i].syncStart(); } _gSetupTool.addResourceToCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "_ANY", 20, STATE_MODEL, RebalanceMode.FULL_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB + "_ANY", _replica); Assert.assertTrue(_clusterVerifier.verifyByPolling()); // Set ANY type throttling after states are stable. setSingleThrottlingConfig(StateTransitionThrottleConfig.RebalanceType.ANY, StateTransitionThrottleConfig.ThrottleScope.INSTANCE, 1); DelayedTransition.setDelay(20); DelayedTransition.enableThrottleRecord(); // start another 3 nodes for (int i = NODE_NR - 3; i < NODE_NR; i++) { _participants[i].syncStart(); } Thread.sleep(2000L); for (int i = 0; i < NODE_NR; i++) { validateThrottle(DelayedTransition.getInstancePatitionTransitionTimes(), _participants[i].getInstanceName(), 1); } }
protected void setupDBs(ClusterSetup clusterSetup) { // Set up target db if (_numDbs > 1) { for (int i = 0; i < _numDbs; i++) { int varyNum = _partitionVary == true ? 10 * i : 0; String db = WorkflowGenerator.DEFAULT_TGT_DB + i; clusterSetup .addResourceToCluster(CLUSTER_NAME, db, _numPartitions + varyNum, MASTER_SLAVE_STATE_MODEL, IdealState.RebalanceMode.FULL_AUTO.toString()); clusterSetup.rebalanceStorageCluster(CLUSTER_NAME, db, _numReplicas); _testDbs.add(db); } } else { if (_instanceGroupTag) { clusterSetup .addResourceToCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, _numPartitions, "OnlineOffline", IdealState.RebalanceMode.FULL_AUTO.name()); IdealState idealState = clusterSetup.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB); idealState.setInstanceGroupTag("TESTTAG0"); clusterSetup.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, idealState); } else { clusterSetup.addResourceToCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, _numPartitions, MASTER_SLAVE_STATE_MODEL, IdealState.RebalanceMode.FULL_AUTO.name()); } clusterSetup.rebalanceStorageCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, _numReplicas); } }
@Test public void testPartitionRecoveryRebalanceThrottle() throws InterruptedException { // start some participants for (int i = 0; i < NODE_NR - 2; i++) { _participants[i].syncStart(); } _gSetupTool.addResourceToCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, 10, STATE_MODEL, RebalanceMode.FULL_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, _replica); Assert.assertTrue(_clusterVerifier.verifyByPolling()); // Set throttling after states are stable. Otherwise it takes too long to reach stable state setSingleThrottlingConfig(StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE, StateTransitionThrottleConfig.ThrottleScope.INSTANCE, 2); DelayedTransition.setDelay(20); DelayedTransition.enableThrottleRecord(); // start another 2 nodes for (int i = NODE_NR - 2; i < NODE_NR; i++) { _participants[i].syncStart(); } Thread.sleep(2000); for (int i = 0; i < NODE_NR; i++) { validateThrottle(DelayedTransition.getInstancePatitionTransitionTimes(), _participants[i].getInstanceName(), 2); } }
@Test public void testTaskThreadCount() throws InterruptedException { String queueName = "myTestJobQueue"; JobQueue.Builder queueBuilder = new JobQueue.Builder(queueName); String lastJob = null; for (int i = 0; i < 5; i++) { String db = TestHelper.getTestMethodName() + "_" + i; _gSetupTool.addResourceToCluster(CLUSTER_NAME, db, 20, MASTER_SLAVE_STATE_MODEL, IdealState.RebalanceMode.FULL_AUTO.name()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db, 1); JobConfig.Builder jobBuilder = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(db) .setNumConcurrentTasksPerInstance(100); queueBuilder.addJob(db + "_job", jobBuilder); lastJob = db + "_job"; } queueBuilder .setWorkflowConfig(new WorkflowConfig.Builder(queueName).setParallelJobs(10).build()); _driver.start(queueBuilder.build()); String nameSpacedJob = TaskUtil.getNamespacedJobName(queueName, lastJob); _driver.pollForJobState(queueName, nameSpacedJob, TaskState.COMPLETED); int threadCountAfter = getThreadCount("TaskStateModelFactory"); Assert.assertTrue( (threadCountAfter - _threadCountBefore) <= TaskStateModelFactory.TASK_THREADPOOL_SIZE + 1); }
@Test (dependsOnMethods = "testParticipantUnavailable") public void testTagSetIncorrect() throws Exception { _gSetupTool.addResourceToCluster(CLUSTER_NAME, testDb, 5, BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO.name()); ZkHelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR) .setResources(new HashSet<>(Collections.singleton(testDb))).build(); Assert.assertTrue(verifier.verifyByPolling()); // Verify there is no rebalance error logged Assert.assertNull(accessor.getProperty(errorNodeKey)); checkRebalanceFailureGauge(false); checkResourceBestPossibleCalFailureState(ResourceMonitor.RebalanceStatus.NORMAL, testDb); // set expected instance tag IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, testDb); is.setInstanceGroupTag("RandomTag"); _gSetupTool.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, testDb, is); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, testDb, 3); // Verify there is rebalance error logged pollForError(accessor, errorNodeKey); checkRebalanceFailureGauge(true); checkResourceBestPossibleCalFailureState( ResourceMonitor.RebalanceStatus.BEST_POSSIBLE_STATE_CAL_FAILED, testDb); // clean up _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME, testDb); }