public static IdealState buildEmptyRealtimeIdealStateFor(String realtimeTableName, int replicaCount, boolean enableBatchMessageMode) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(realtimeTableName); customModeIdealStateBuilder .setStateModel(PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL) .setNumPartitions(0).setNumReplica(replicaCount).setMaxPartitionsPerNode(1); final IdealState idealState = customModeIdealStateBuilder.build(); idealState.setInstanceGroupTag(realtimeTableName); idealState.setBatchMessageMode(enableBatchMessageMode); return idealState; }
/** * * Building an empty idealState for a given table. * Used when creating a new table. * * @param tableName resource name * @param numCopies is the number of replicas * @return */ public static IdealState buildEmptyIdealStateFor(String tableName, int numCopies, boolean enableBatchMessageMode) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(tableName); final int replicas = numCopies; customModeIdealStateBuilder .setStateModel(PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL) .setNumPartitions(0).setNumReplica(replicas).setMaxPartitionsPerNode(1); final IdealState idealState = customModeIdealStateBuilder.build(); idealState.setInstanceGroupTag(tableName); idealState.setBatchMessageMode(enableBatchMessageMode); return idealState; }
.setNumPartitions(0).setNumReplica(nReplicas).setMaxPartitionsPerNode(1); IdealState idealState = customModeIdealStateBuilder.build(); idealState.setInstanceGroupTag(tableName);
.setNumPartitions(0).setNumReplica(nReplicas).setMaxPartitionsPerNode(1); IdealState idealState = customModeIdealStateBuilder.build(); idealState.setInstanceGroupTag(offlineTableName); setInstanceStateMapForIdealStateOffline(idealState, nSegments, nReplicas, instances, offlineTableName);
.setNumPartitions(0).setNumReplica(nReplicas).setMaxPartitionsPerNode(1); IdealState idealState = customModeIdealStateBuilder.build(); idealState.setInstanceGroupTag(realtimeTableName); setInstanceStateMapForIdealStateRealtimeCompleted(idealState, nPartitions, nIterationsCompleted, nReplicas, completedInstances, realtimeTableName);
idealState.setInstanceGroupTag(groupId);
idealState.setInstanceGroupTag(groupId);
/** * Create an IdealState for a resource that belongs to a resource group We use * "resourceGroupName$resourceInstanceTag" as the IdealState znode name to differetiate different * resources from the same resourceGroup. */ public IdealState createIdealStateForResourceGroup(String resourceGroupName, String resourceTag, int numPartition, int replica, String rebalanceMode, String stateModelDefName) { String idealStateId = genIdealStateNameWithResourceTag(resourceGroupName, resourceTag); IdealState idealState = new IdealState(idealStateId); idealState.setNumPartitions(numPartition); idealState.setStateModelDefRef(stateModelDefName); IdealState.RebalanceMode mode = idealState.rebalanceModeFromString(rebalanceMode, IdealState.RebalanceMode.SEMI_AUTO); idealState.setRebalanceMode(mode); idealState.setReplicas("" + replica); idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY); idealState.setResourceGroupName(resourceGroupName); idealState.setInstanceGroupTag(resourceTag); idealState.enableGroupRouting(true); return idealState; }
/** * Create an IdealState for a resource that belongs to a resource group We use * "resourceGroupName$resourceInstanceTag" as the IdealState znode name to differetiate different * resources from the same resourceGroup. */ public IdealState createIdealStateForResourceGroup(String resourceGroupName, String resourceTag, int numPartition, int replica, String rebalanceMode, String stateModelDefName) { String idealStateId = genIdealStateNameWithResourceTag(resourceGroupName, resourceTag); IdealState idealState = new IdealState(idealStateId); idealState.setNumPartitions(numPartition); idealState.setStateModelDefRef(stateModelDefName); IdealState.RebalanceMode mode = idealState.rebalanceModeFromString(rebalanceMode, IdealState.RebalanceMode.SEMI_AUTO); idealState.setRebalanceMode(mode); idealState.setReplicas("" + replica); idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY); idealState.setResourceGroupName(resourceGroupName); idealState.setInstanceGroupTag(resourceTag); idealState.enableGroupRouting(true); return idealState; }
idealState.setInstanceGroupTag(TAG); helixAdmin.setResourceIdealState(clusterName, RESOURCE_NAME, idealState);
@Test(dataProvider = "rebalanceStrategies") public void testZoneIsolationWithInstanceTag( String rebalanceStrategyName, String rebalanceStrategyClass) throws Exception { Set<String> tags = new HashSet<String>(_nodeToTagMap.values()); int i = 0; for (String tag : tags) { String db = "Test-DB-Tag-" + rebalanceStrategyName + "-" + i++; _gSetupTool.addResourceToCluster(CLUSTER_NAME, db, _PARTITIONS, BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO + "", rebalanceStrategyClass); IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); is.setInstanceGroupTag(tag); _gSetupTool.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, db, is); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db, _replica); _allDBs.add(db); } Thread.sleep(300); ZkHelixClusterVerifier _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR) .setResources(_allDBs).build(); Assert.assertTrue(_clusterVerifier.verifyByPolling()); for (String db : _allDBs) { IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); ExternalView ev = _gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db); validateZoneAndTagIsolation(is, ev, _replica); } }
@Test(dataProvider = "rebalanceStrategies", enabled = true, dependsOnMethods = "test") public void testWithInstanceTag(String rebalanceStrategyName, String rebalanceStrategyClass) throws Exception { Set<String> tags = new HashSet<String>(_nodeToTagMap.values()); int i = 3; for (String tag : tags) { String db = "Test-DB-" + rebalanceStrategyName + "-" + i++; _gSetupTool.addResourceToCluster(CLUSTER_NAME, db, _PARTITIONS, BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO + "", rebalanceStrategyClass); IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); is.setInstanceGroupTag(tag); _gSetupTool.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, db, is); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db, _replica); _allDBs.add(db); } Thread.sleep(300); HelixClusterVerifier _clusterVerifier = new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR) .setResources(_allDBs).build(); Assert.assertTrue(_clusterVerifier.verify(5000)); for (String db : _allDBs) { IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); ExternalView ev = _gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db); validateIsolation(is, ev, _replica); } }
idealState.setInstanceGroupTag(TAG); helixAdmin.setResourceIdealState(clusterName, RESOURCE_NAME, idealState);
@Test (dependsOnMethods = "testParticipantUnavailable") public void testTagSetIncorrect() throws Exception { _gSetupTool.addResourceToCluster(CLUSTER_NAME, testDb, 5, BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO.name()); ZkHelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR) .setResources(new HashSet<>(Collections.singleton(testDb))).build(); Assert.assertTrue(verifier.verifyByPolling()); // Verify there is no rebalance error logged Assert.assertNull(accessor.getProperty(errorNodeKey)); checkRebalanceFailureGauge(false); checkResourceBestPossibleCalFailureState(ResourceMonitor.RebalanceStatus.NORMAL, testDb); // set expected instance tag IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, testDb); is.setInstanceGroupTag("RandomTag"); _gSetupTool.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, testDb, is); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, testDb, 3); // Verify there is rebalance error logged pollForError(accessor, errorNodeKey); checkRebalanceFailureGauge(true); checkResourceBestPossibleCalFailureState( ResourceMonitor.RebalanceStatus.BEST_POSSIBLE_STATE_CAL_FAILED, testDb); // clean up _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME, testDb); }
idealState.setInstanceGroupTag(TAG); helixAdmin.setResourceIdealState(clusterName, RESOURCE_NAME, idealState);
protected void setupDBs(ClusterSetup clusterSetup) { // Set up target db if (_numDbs > 1) { for (int i = 0; i < _numDbs; i++) { int varyNum = _partitionVary == true ? 10 * i : 0; String db = WorkflowGenerator.DEFAULT_TGT_DB + i; clusterSetup .addResourceToCluster(CLUSTER_NAME, db, _numPartitions + varyNum, MASTER_SLAVE_STATE_MODEL, IdealState.RebalanceMode.FULL_AUTO.toString()); clusterSetup.rebalanceStorageCluster(CLUSTER_NAME, db, _numReplicas); _testDbs.add(db); } } else { if (_instanceGroupTag) { clusterSetup .addResourceToCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, _numPartitions, "OnlineOffline", IdealState.RebalanceMode.FULL_AUTO.name()); IdealState idealState = clusterSetup.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB); idealState.setInstanceGroupTag("TESTTAG0"); clusterSetup.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, idealState); } else { clusterSetup.addResourceToCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, _numPartitions, MASTER_SLAVE_STATE_MODEL, IdealState.RebalanceMode.FULL_AUTO.name()); } clusterSetup.rebalanceStorageCluster(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, _numReplicas); } }
is.enable(true); if (i < 2) { is.setInstanceGroupTag(TEST_TAG);