protected void setTableIdealState(String realtimeTableName, IdealState idealState) { _helixAdmin.setResourceIdealState(_clusterName, realtimeTableName, idealState); }
private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) { IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); for (String tableNameWithType : tableIdealState.getPartitionSet()) { TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); String brokerTag = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); if (brokerTag.equals(brokerTenantTag)) { tableIdealState.setPartitionState(tableNameWithType, instanceName, BrokerOnlineOfflineStateModel.ONLINE); } } _helixAdmin .setResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, tableIdealState); }
IdealState idealState = PinotTableIdealStateBuilder .buildEmptyIdealStateForBrokerResource(admin, helixClusterName, enableBatchMessageMode); admin.setResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState); initPropertyStorePath(helixClusterName, zkPath); LOGGER.info("New Cluster setup completed... ********************************************** ");
private static void assignIdealStates(HelixManager helixManager, Map<String, IdealState> idealStatesFromAssignment) { HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : idealStatesFromAssignment.keySet()) { IdealState idealState = idealStatesFromAssignment.get(topic); helixAdmin.setResourceIdealState(helixClusterName, topic, idealState); } }
public synchronized void expandTopicInMirrorMaker(String topicName, int newNumTopicPartitions) { updateCurrentServingInstance(); synchronized (_currentServingInstance) { _helixAdmin.setResourceIdealState(_helixClusterName, topicName, IdealStateBuilder.expandCustomRebalanceModeIdealStateFor( _helixAdmin.getResourceIdealState(_helixClusterName, topicName), topicName, newNumTopicPartitions, _currentServingInstance)); } }
private void assignIdealStates(HelixManager helixManager, Map<String, IdealState> idealStatesFromAssignment) { HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : idealStatesFromAssignment.keySet()) { IdealState idealState = idealStatesFromAssignment.get(topic); helixAdmin.setResourceIdealState(helixClusterName, topic, idealState); } }
public synchronized void replaceWorkerInMirrorMaker(Map<String, List<String>> pipelineToRouteIdToReplace, List<String> workerToReplace) { _lock.lock(); try { LOGGER.info("replace: {}", _availableWorkerList); for (String pipeline : pipelineToRouteIdToReplace.keySet()) { LOGGER.info("replace: {} : {}", pipeline, pipelineToRouteIdToReplace.get(pipeline)); // TODO: if there aren't enough workers _helixAdmin.setResourceIdealState(_helixClusterName, pipeline, IdealStateBuilder.resetCustomIdealStateFor(_helixAdmin.getResourceIdealState(_helixClusterName, pipeline), pipeline, workerToReplace, _availableWorkerList, _conf.getMaxNumWorkersPerRoute())); } } finally { _lock.unlock(); } }
public synchronized void expandTopicInMirrorMaker(String topicName, int newNumTopicPartitions) { updateCurrentServingInstance(); synchronized (_currentServingInstance) { _helixAdmin.setResourceIdealState(_helixClusterName, topicName, IdealStateBuilder.expandCustomRebalanceModeIdealStateFor( _helixAdmin.getResourceIdealState(_helixClusterName, topicName), topicName, newNumTopicPartitions, _controllerConf, _currentServingInstance)); } }
public void removeResourceProperty(String clusterName, String resourceName, String propertyKey) { IdealState idealState = _admin.getResourceIdealState(clusterName, resourceName); if (idealState == null) { throw new HelixException("Resource: " + resourceName + " has NOT been added yet"); } idealState.getRecord().getSimpleFields().remove(propertyKey); _admin.setResourceIdealState(clusterName, resourceName, idealState); }
public void removeResourceProperty(String clusterName, String resourceName, String propertyKey) { IdealState idealState = _admin.getResourceIdealState(clusterName, resourceName); if (idealState == null) { throw new HelixException("Resource: " + resourceName + " has NOT been added yet"); } idealState.getRecord().getSimpleFields().remove(propertyKey); _admin.setResourceIdealState(clusterName, resourceName, idealState); }
public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException { ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile))); if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) { throw new IllegalArgumentException("ideal state must have same id as resource name"); } _admin.setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord)); }
public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException { ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile))); if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) { throw new IllegalArgumentException("ideal state must have same id as resource name"); } _admin.setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord)); }
@Override public void onIdealStateChange(List<IdealState> list, NotificationContext notificationContext) { HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); for (String resource : helixAdmin.getResourcesInCluster(clusterName)) { if (resource.equals(jobResourceName)) { IdealState idealState = helixAdmin.getResourceIdealState(clusterName, resource); if (idealState != null) { String rebalancerClassFound = idealState.getRebalancerClassName(); if (rebalancerToReplace.equals(rebalancerClassFound)) { idealState.setRebalancerClassName(rebalancerClassDesired); helixAdmin.setResourceIdealState(clusterName, resource, idealState); } } } } } });
/** Posts new workflow resource to cluster */ private void addWorkflowResource(String workflow) { // Add workflow resource _admin.addResource(_clusterName, workflow, 1, TaskConstants.STATE_MODEL_NAME); IdealState is = buildWorkflowIdealState(workflow); TaskUtil .createUserContent(_propertyStore, workflow, new ZNRecord(TaskUtil.USER_CONTENT_NODE)); _admin.setResourceIdealState(_clusterName, workflow, is); }
public void addResourceProperty(String clusterName, String resourceName, String propertyKey, String propertyVal) { IdealState idealState = _admin.getResourceIdealState(clusterName, resourceName); if (idealState == null) { throw new HelixException("Resource: " + resourceName + " has NOT been added yet"); } idealState.getRecord().setSimpleField(propertyKey, propertyVal); _admin.setResourceIdealState(clusterName, resourceName, idealState); }
/** Posts new workflow resource to cluster */ private void addWorkflowResource(String workflow) { // Add workflow resource _admin.addResource(_clusterName, workflow, 1, TaskConstants.STATE_MODEL_NAME); IdealState is = buildWorkflowIdealState(workflow); TaskUtil .createUserContent(_propertyStore, workflow, new ZNRecord(TaskUtil.USER_CONTENT_NODE)); _admin.setResourceIdealState(_clusterName, workflow, is); }
public void addResourceProperty(String clusterName, String resourceName, String propertyKey, String propertyVal) { IdealState idealState = _admin.getResourceIdealState(clusterName, resourceName); if (idealState == null) { throw new HelixException("Resource: " + resourceName + " has NOT been added yet"); } idealState.getRecord().setSimpleField(propertyKey, propertyVal); _admin.setResourceIdealState(clusterName, resourceName, idealState); }
private void setupUnbalancedDB() throws InterruptedException { // Start with Full-Auto mode to create the partitions, Semi-Auto won't create partitions. _gSetupTool.addResourceToCluster(CLUSTER_NAME, UNBALANCED_DB_NAME, 50, MASTER_SLAVE_STATE_MODEL, IdealState.RebalanceMode.FULL_AUTO.toString()); _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, UNBALANCED_DB_NAME, 1); // Set preference list to put all partitions to one instance. IdealState idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, UNBALANCED_DB_NAME); Set<String> partitions = idealState.getPartitionSet(); for (String partition : partitions) { idealState.setPreferenceList(partition, Lists.newArrayList(_blockedParticipant.getInstanceName())); } idealState.setRebalanceMode(IdealState.RebalanceMode.SEMI_AUTO); _gSetupTool.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, UNBALANCED_DB_NAME, idealState); Assert.assertTrue(_clusterVerifier.verifyByPolling(10000, 100)); } }
protected void createDBInSemiAuto(ClusterSetup clusterSetup, String clusterName, String dbName, List<String> preferenceList, String stateModelDef, int numPartition, int replica) { clusterSetup.addResourceToCluster(clusterName, dbName, numPartition, stateModelDef, IdealState.RebalanceMode.SEMI_AUTO.toString()); clusterSetup.rebalanceStorageCluster(clusterName, dbName, replica); IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(clusterName, dbName); for (String p : is.getPartitionSet()) { is.setPreferenceList(p, preferenceList); } clusterSetup.getClusterManagementTool().setResourceIdealState(clusterName, dbName, is); }
protected IdealState createResourceWithDelayedRebalance(String clusterName, String db, String stateModel, int numPartition, int replica, int minActiveReplica, long delay, String rebalanceStrategy) { IdealState idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(clusterName, db); if (idealState == null) { _gSetupTool.addResourceToCluster(clusterName, db, numPartition, stateModel, IdealState.RebalanceMode.FULL_AUTO + "", rebalanceStrategy); } idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(clusterName, db); idealState.setMinActiveReplicas(minActiveReplica); if (!idealState.isDelayRebalanceEnabled()) { idealState.setDelayRebalanceEnabled(true); } if (delay > 0) { idealState.setRebalanceDelay(delay); } idealState.setRebalancerClassName(DelayedAutoRebalancer.class.getName()); _gSetupTool.getClusterManagementTool().setResourceIdealState(clusterName, db, idealState); _gSetupTool.rebalanceStorageCluster(clusterName, db, replica); idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(clusterName, db); return idealState; }