@Nullable @Override public IdealState apply(@Nullable IdealState idealState) { if (!idealState.isEnabled()) { LOGGER.info("Skipping relocation of segments for {} since ideal state is disabled", tableNameWithType); return null; } relocateSegments(realtimeTagConfig, idealState); return idealState; } };
if (idealState == null || !idealState.isEnabled()) { iterator.remove(); continue;
private Set<String> fetchLatestTableResources(HelixAdmin helixAdmin) { Set<String> resourcesToMonitor = new HashSet<>(); for (String resourceName : helixAdmin.getResourcesInCluster(_helixClusterName)) { // Only monitor table resources if (!TableNameBuilder.isTableResource(resourceName)) { continue; } // Only monitor enabled resources IdealState idealState = helixAdmin.getResourceIdealState(_helixClusterName, resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceId)) { resourcesToMonitor.add(resourceName); break; } } } } return resourcesToMonitor; }
public IdealStateMatchServiceStatusCallback(HelixManager helixManager, String clusterName, String instanceName) { _clusterName = clusterName; _instanceName = instanceName; _helixAdmin = helixManager.getClusterManagmentTool(); _helixDataAccessor = helixManager.getHelixDataAccessor(); _resourcesToMonitor = new HashSet<>(); for (String resourceName : _helixAdmin.getResourcesInCluster(_clusterName)) { // Only monitor table resources and broker resource if (!TableNameBuilder.isTableResource(resourceName) && !resourceName .equals(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)) { continue; } // Only monitor enabled resources IdealState idealState = getResourceIdealState(resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceName)) { _resourcesToMonitor.add(resourceName); break; } } } } _numTotalResourcesToMonitor = _resourcesToMonitor.size(); LOGGER.info("Monitoring {} resources: {} for start up of instance {}", _numTotalResourcesToMonitor, _resourcesToMonitor, _instanceName); }
/** * Sets up a new table's segments metadata and returns the ideal state setup with initial segments * @param tableConfig * @param idealState * @param partitionCount * @return */ private IdealState setupTable(TableConfig tableConfig, IdealState idealState, int partitionCount) throws InvalidConfigException { final String tableNameWithType = tableConfig.getTableName(); if (!idealState.isEnabled()) { LOGGER.info("Skipping validation for disabled table {}", tableNameWithType); return idealState; } final StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); final long now = getCurrentTimeMs(); PartitionAssignment partitionAssignment = _streamPartitionAssignmentGenerator.generateStreamPartitionAssignment(tableConfig, partitionCount); Set<Integer> newPartitions = new HashSet<>(partitionCount); for (int partition = 0; partition < partitionCount; partition++) { newPartitions.add(partition); } OffsetCriteria offsetCriteria = streamConfig.getOffsetCriteria(); Set<String> consumingSegments = setupNewPartitions(tableConfig, streamConfig, offsetCriteria, partitionAssignment, newPartitions, now); RealtimeSegmentAssignmentStrategy segmentAssignmentStrategy = new ConsumingSegmentAssignmentStrategy(); Map<String, List<String>> assignments = segmentAssignmentStrategy.assign(consumingSegments, partitionAssignment); updateIdealState(idealState, null, consumingSegments, assignments); return idealState; }
if (!idealState.isEnabled()) { if (_logDisabledTables) { LOGGER.warn("Table {} is disabled. Skipping segment status checks", tableNameWithType);
if (!skipNewPartitions && idealState.isEnabled()) { Assert.assertEquals(allSegmentsForPartition.size(), 1); for (String segment : allSegmentsForPartition) {
final String tableNameWithType = tableConfig.getTableName(); final StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); if (!idealState.isEnabled()) { LOGGER.info("Skipping validation for disabled table {}", tableNameWithType); return idealState;
segmentManager._partitionAssignmentGenerator.getPartitionToLatestSegments(idealState); Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); if (!idealState.isEnabled()) { Assert.assertTrue(partitionToLatestSegments.isEmpty()); } else {
if (!idealState.isEnabled()) { return computeBestPossibleMapForDisabledResource(currentStateMap, stateModelDef);
if (!idealState.isEnabled()) { return computeBestPossibleMapForDisabledResource(currentStateMap, stateModelDef);
protected Map<String, String> computeBestPossibleStateForPartition(Set<String> liveInstances, StateModelDefinition stateModelDef, List<String> preferenceList, CurrentStateOutput currentStateOutput, Set<String> disabledInstancesForPartition, IdealState idealState, ClusterConfig clusterConfig, Partition partition) { Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(idealState.getResourceName(), partition); if (currentStateMap == null) { currentStateMap = Collections.emptyMap(); } // (1) If the partition is removed from IS or the IS is deleted. // Transit to DROPPED no matter the instance is disabled or not. if (preferenceList == null) { return computeBestPossibleMapForDroppedResource(currentStateMap); } // (2) If resource disabled altogether, transit to initial-state (e.g. OFFLINE) if it's not in ERROR. if (!idealState.isEnabled()) { return computeBestPossibleMapForDisabledResource(currentStateMap, stateModelDef); } return computeBestPossibleMap(preferenceList, stateModelDef, currentStateMap, liveInstances, disabledInstancesForPartition); }
protected Map<String, String> computeBestPossibleStateForPartition(Set<String> liveInstances, StateModelDefinition stateModelDef, List<String> preferenceList, CurrentStateOutput currentStateOutput, Set<String> disabledInstancesForPartition, IdealState idealState, ClusterConfig clusterConfig, Partition partition) { Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(idealState.getResourceName(), partition); if (currentStateMap == null) { currentStateMap = Collections.emptyMap(); } // (1) If the partition is removed from IS or the IS is deleted. // Transit to DROPPED no matter the instance is disabled or not. if (preferenceList == null) { return computeBestPossibleMapForDroppedResource(currentStateMap); } // (2) If resource disabled altogether, transit to initial-state (e.g. OFFLINE) if it's not in ERROR. if (!idealState.isEnabled()) { return computeBestPossibleMapForDisabledResource(currentStateMap, stateModelDef); } return computeBestPossibleMap(preferenceList, stateModelDef, currentStateMap, liveInstances, disabledInstancesForPartition); }
Map<String, String> bestStateForPartition = computeCustomizedBestStateForPartition(cache, stateModelDef, idealStateMap, currentStateMap, disabledInstancesForPartition, idealState.isEnabled()); partitionMapping.addReplicaMap(partition, bestStateForPartition);
Map<String, String> bestStateForPartition = computeCustomizedBestStateForPartition(cache, stateModelDef, idealStateMap, currentStateMap, disabledInstancesForPartition, idealState.isEnabled()); partitionMapping.addReplicaMap(partition, bestStateForPartition);
@Test public void testDisableResource() { String className = TestHelper.getTestClassName(); String methodName = TestHelper.getTestMethodName(); String clusterName = className + "_" + methodName; System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis())); HelixAdmin admin = new ZKHelixAdmin(_gZkClient); admin.addCluster(clusterName, true); Assert.assertTrue(ZKUtil.isClusterSetup(clusterName, _gZkClient), "Cluster should be setup"); String resourceName = "TestDB"; admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition( StateModelConfigGenerator.generateConfigForMasterSlave())); admin.addResource(clusterName, resourceName, 4, "MasterSlave"); admin.enableResource(clusterName, resourceName, false); BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient); HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); IdealState idealState = accessor.getProperty(keyBuilder.idealStates(resourceName)); Assert.assertFalse(idealState.isEnabled()); admin.enableResource(clusterName, resourceName, true); idealState = accessor.getProperty(keyBuilder.idealStates(resourceName)); Assert.assertTrue(idealState.isEnabled()); admin.dropCluster(clusterName); System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); }
|| (jobCfg.getTargetResource() != null && cache.getIdealState(jobCfg.getTargetResource()) != null && !cache.getIdealState(jobCfg.getTargetResource()).isEnabled())) { if (isJobFinished(jobCtx, jobResource, currStateOutput)) { failJob(jobResource, workflowCtx, jobCtx, workflowConfig, cache.getJobConfigMap(), cache);
|| (jobCfg.getTargetResource() != null && cache.getIdealState(jobCfg.getTargetResource()) != null && !cache.getIdealState(jobCfg.getTargetResource()).isEnabled())) { if (isJobFinished(jobCtx, jobResource, currStateOutput)) { failJob(jobResource, workflowCtx, jobCtx, workflowConfig, cache.getJobConfigMap(), cache);