public GobblinHelixTaskDriver(HelixManager manager) { this(manager.getClusterManagmentTool(), manager.getHelixDataAccessor(), manager .getConfigAccessor(), manager.getHelixPropertyStore(), manager.getClusterName()); }
private void checkRebalancingTimer(HelixManager manager, List<IdealState> idealStates, ClusterConfig clusterConfig) { if (manager.getConfigAccessor() == null) { logger.warn(manager.getInstanceName() + " config accessor doesn't exist. should be in file-based mode."); return; } long minPeriod = Long.MAX_VALUE; if (clusterConfig != null) { long period = clusterConfig.getRebalanceTimePeriod(); if (period > 0 && minPeriod > period) { minPeriod = period; } } // TODO: resource level rebalance does not make sense, to remove it! for (IdealState idealState : idealStates) { long period = idealState.getRebalanceTimerPeriod(); if (period > 0 && minPeriod > period) { minPeriod = period; } } if (minPeriod != Long.MAX_VALUE) { startRebalancingTimer(minPeriod, manager); } }
private static void markForDeletion(HelixManager mgr, String resourceName) { mgr.getConfigAccessor().set( TaskUtil.getResourceConfigScope(mgr.getClusterName(), resourceName), WorkflowConfig.WorkflowConfigProperty.TargetState.name(), TargetState.DELETE.name()); }
private static void markForDeletion(HelixManager mgr, String resourceName) { mgr.getConfigAccessor().set( TaskUtil.getResourceConfigScope(mgr.getClusterName(), resourceName), WorkflowConfig.WorkflowConfigProperty.TargetState.name(), TargetState.DELETE.name()); }
private void checkRebalancingTimer(HelixManager manager, List<IdealState> idealStates, ClusterConfig clusterConfig) { if (manager.getConfigAccessor() == null) { logger.warn(manager.getInstanceName() + " config accessor doesn't exist. should be in file-based mode."); return; } long minPeriod = Long.MAX_VALUE; if (clusterConfig != null) { long period = clusterConfig.getRebalanceTimePeriod(); if (period > 0 && minPeriod > period) { minPeriod = period; } } // TODO: resource level rebalance does not make sense, to remove it! for (IdealState idealState : idealStates) { long period = idealState.getRebalanceTimerPeriod(); if (period > 0 && minPeriod > period) { minPeriod = period; } } if (minPeriod != Long.MAX_VALUE) { startPeriodRebalance(minPeriod, manager); } else { stopPeriodRebalance(); } }
public ParticipantManager(HelixManager manager, HelixZkClient zkclient, int sessionTimeout, LiveInstanceInfoProvider liveInstanceInfoProvider, List<PreConnectCallback> preConnectCallbacks) { _zkclient = zkclient; _manager = manager; _clusterName = manager.getClusterName(); _instanceName = manager.getInstanceName(); _keyBuilder = new PropertyKey.Builder(_clusterName); _sessionId = manager.getSessionId(); _sessionTimeout = sessionTimeout; _configAccessor = manager.getConfigAccessor(); _instanceType = manager.getInstanceType(); _helixAdmin = manager.getClusterManagmentTool(); _dataAccessor = (ZKHelixDataAccessor) manager.getHelixDataAccessor(); _messagingService = (DefaultMessagingService) manager.getMessagingService(); _stateMachineEngine = manager.getStateMachineEngine(); _liveInstanceInfoProvider = liveInstanceInfoProvider; _preConnectCallbacks = preConnectCallbacks; }
public ParticipantManager(HelixManager manager, ZkClient zkclient, int sessionTimeout, LiveInstanceInfoProvider liveInstanceInfoProvider, List<PreConnectCallback> preConnectCallbacks) { _zkclient = zkclient; _manager = manager; _clusterName = manager.getClusterName(); _instanceName = manager.getInstanceName(); _keyBuilder = new PropertyKey.Builder(_clusterName); _sessionId = manager.getSessionId(); _sessionTimeout = sessionTimeout; _configAccessor = manager.getConfigAccessor(); _instanceType = manager.getInstanceType(); _helixAdmin = manager.getClusterManagmentTool(); _dataAccessor = (ZKHelixDataAccessor) manager.getHelixDataAccessor(); _messagingService = (DefaultMessagingService) manager.getMessagingService(); _stateMachineEngine = manager.getStateMachineEngine(); _liveInstanceInfoProvider = liveInstanceInfoProvider; _preConnectCallbacks = preConnectCallbacks; }
String key = type + "." + HelixTaskExecutor.MAX_THREADS; ConfigAccessor configAccessor = _manager.getConfigAccessor(); if (configAccessor != null) { ConfigScope scope = null;
String key = type + "." + HelixTaskExecutor.MAX_THREADS; ConfigAccessor configAccessor = _manager.getConfigAccessor(); if (configAccessor != null) { ConfigScope scope = null;
ClusterConfig clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME); clusterConfig.resetTaskQuotaRatioMap(); clusterConfig.setTaskQuotaRatio(DEFAULT_QUOTA_TYPE, 1); clusterConfig.setTaskQuotaRatio("A", 1); _manager.getConfigAccessor().setClusterConfig(CLUSTER_NAME, clusterConfig); _availableQuotaTypes = clusterConfig.getTaskQuotaRatioMap().keySet();
ClusterConfig clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME);
ClusterConfig clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME); clusterConfig.resetTaskQuotaRatioMap(); clusterConfig.setTaskQuotaRatio(DEFAULT_QUOTA_TYPE, 1); clusterConfig.setTaskQuotaRatio("A", 1); clusterConfig.setTaskQuotaRatio("B", 1); _manager.getConfigAccessor().setClusterConfig(CLUSTER_NAME, clusterConfig); _availableQuotaTypes = clusterConfig.getTaskQuotaRatioMap().keySet();
private void setResourceThreadPoolSize(String resourceName, int threadPoolSize) { HelixManager manager = _participants[0]; ConfigAccessor accessor = manager.getConfigAccessor(); HelixConfigScope scope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.RESOURCE) .forCluster(manager.getClusterName()).forResource(resourceName).build(); accessor.set(scope, HelixTaskExecutor.MAX_THREADS, "" + threadPoolSize); }
@Test(dependsOnMethods = "testSchedulingWithoutQuota") public void testQuotaConfigChange() throws InterruptedException { ClusterConfig clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME); clusterConfig.resetTaskQuotaRatioMap(); clusterConfig.setTaskQuotaRatio(DEFAULT_QUOTA_TYPE, 38); clusterConfig.setTaskQuotaRatio("A", 1); clusterConfig.setTaskQuotaRatio("B", 1); _manager.getConfigAccessor().setClusterConfig(CLUSTER_NAME, clusterConfig); clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME); clusterConfig.resetTaskQuotaRatioMap(); clusterConfig.setTaskQuotaRatio(DEFAULT_QUOTA_TYPE, 1); clusterConfig.setTaskQuotaRatio("A", 38); clusterConfig.setTaskQuotaRatio("B", 1); _manager.getConfigAccessor().setClusterConfig(CLUSTER_NAME, clusterConfig);
ClusterConfig clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME); clusterConfig.resetTaskQuotaRatioMap(); clusterConfig.setTaskQuotaRatio(DEFAULT_QUOTA_TYPE, 1); clusterConfig.setTaskQuotaRatio("A", 1); clusterConfig.setTaskQuotaRatio("B", 1); _manager.getConfigAccessor().setClusterConfig(CLUSTER_NAME, clusterConfig); _availableQuotaTypes = clusterConfig.getTaskQuotaRatioMap().keySet();
HelixManager manager = _participants[0]; ConfigAccessor accessor = manager.getConfigAccessor(); ConfigScope scope = new ConfigScopeBuilder().forCluster(manager.getClusterName()).forParticipant(instanceName)
public void testJobQueueScheduling() throws InterruptedException { ClusterConfig clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME); clusterConfig.resetTaskQuotaRatioMap(); clusterConfig.setTaskQuotaRatio(DEFAULT_QUOTA_TYPE, 1); clusterConfig.setTaskQuotaRatio("A", 1); clusterConfig.setTaskQuotaRatio("B", 1); _manager.getConfigAccessor().setClusterConfig(CLUSTER_NAME, clusterConfig); _availableQuotaTypes = clusterConfig.getTaskQuotaRatioMap().keySet();
@BeforeClass public void beforeClass() throws Exception { _gSetupTool.addCluster(CLUSTER_NAME, true); _participants = new MockParticipantManager[NUM_NODES]; for (int i = 0; i < NUM_NODES; i++) { String storageNodeName = PARTICIPANT_PREFIX + "_" + (START_PORT + i); _gSetupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName); } for (int i = 0; i < NUM_NODES; i++) { String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i); _participants[i] = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instanceName); _participants[i].syncStart(); } _manager = HelixManagerFactory .getZKHelixManager(CLUSTER_NAME, "Admin", InstanceType.ADMINISTRATOR, ZK_ADDR); _manager.connect(); String controllerName = CONTROLLER_PREFIX + "_0"; _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, controllerName); _controller.syncStart(); ConfigAccessor _configAccessor = _manager.getConfigAccessor(); ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME); clusterConfig.enableTargetExternalView(true); _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig); }
manager.getConfigAccessor().set(scope, ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, "true");
ClusterConfig clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME); clusterConfig.resetTaskQuotaRatioMap(); clusterConfig.setTaskQuotaRatio(DEFAULT_QUOTA_TYPE, 10); clusterConfig.setTaskQuotaRatio("A", 10); clusterConfig.setTaskQuotaRatio("B", 10); _manager.getConfigAccessor().setClusterConfig(CLUSTER_NAME, clusterConfig); _availableQuotaTypes = clusterConfig.getTaskQuotaRatioMap().keySet();