/*** * Build a Helix Manager (Helix Controller instance). * * @param helixInstanceName the Helix Instance name. * @param helixClusterName the Helix Cluster name. * @param zkConnectionString the ZooKeeper connection string. * @return HelixManager */ public static HelixManager buildHelixManager(String helixInstanceName, String helixClusterName, String zkConnectionString) { return HelixManagerFactory.getZKHelixManager(helixClusterName, helixInstanceName, InstanceType.CONTROLLER, zkConnectionString); }
/** * Build the {@link HelixManager} for the Application Master. */ protected static HelixManager buildHelixManager(Config config, String zkConnectionString, String clusterName, InstanceType type) { String helixInstanceName = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, GobblinClusterManager.class.getSimpleName()); return HelixManagerFactory.getZKHelixManager( config.getString(clusterName), helixInstanceName, type, zkConnectionString); }
private void initHelixManager() { String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); logger.info("Using ZooKeeper connection string: " + zkConnectionString); if (this.isTaskDriver && this.dedicatedTaskDriverCluster) { // This will create a Helix manager to receive the planning job this.taskDriverHelixManager = Optional.of(HelixManagerFactory.getZKHelixManager( ConfigUtils.getString(this.config, GobblinClusterConfigurationKeys.TASK_DRIVER_CLUSTER_NAME_KEY, ""), this.helixInstanceName, InstanceType.PARTICIPANT, zkConnectionString)); this.jobHelixManager = HelixManagerFactory.getZKHelixManager( this.clusterName, this.helixInstanceName, InstanceType.ADMINISTRATOR, zkConnectionString); } else { this.jobHelixManager = HelixManagerFactory.getZKHelixManager( this.clusterName, this.helixInstanceName, InstanceType.PARTICIPANT, zkConnectionString); } }
public GobblinYarnAppLauncher(Config config, YarnConfiguration yarnConfiguration) throws IOException { this.config = config; this.applicationName = config.getString(GobblinYarnConfigurationKeys.APPLICATION_NAME_KEY); this.appQueueName = config.getString(GobblinYarnConfigurationKeys.APP_QUEUE_KEY); String zkConnectionString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); LOGGER.info("Using ZooKeeper connection string: " + zkConnectionString); this.helixManager = HelixManagerFactory.getZKHelixManager( config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), GobblinClusterUtils.getHostname(), InstanceType.SPECTATOR, zkConnectionString); this.yarnConfiguration = yarnConfiguration; this.yarnConfiguration.set("fs.automatic.close", "false"); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(this.yarnConfiguration); this.fs = config.hasPath(ConfigurationKeys.FS_URI_KEY) ? FileSystem.get(URI.create(config.getString(ConfigurationKeys.FS_URI_KEY)), this.yarnConfiguration) : FileSystem.get(this.yarnConfiguration); this.closer.register(this.fs); this.applicationStatusMonitor = Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("GobblinYarnAppStatusMonitor"))); this.appReportIntervalMinutes = config.getLong(GobblinYarnConfigurationKeys.APP_REPORT_INTERVAL_MINUTES_KEY); this.appMasterJvmArgs = config.hasPath(GobblinYarnConfigurationKeys.APP_MASTER_JVM_ARGS_KEY) ? Optional.of(config.getString(GobblinYarnConfigurationKeys.APP_MASTER_JVM_ARGS_KEY)) : Optional.<String>absent(); this.sinkLogRootDir = new Path(config.getString(GobblinYarnConfigurationKeys.LOGS_SINK_ROOT_DIR_KEY)); this.maxGetApplicationReportFailures = config.getInt(GobblinYarnConfigurationKeys.MAX_GET_APP_REPORT_FAILURES_KEY); this.emailNotificationOnShutdown = config.getBoolean(GobblinYarnConfigurationKeys.EMAIL_NOTIFICATION_ON_SHUTDOWN_KEY); }
HelixManagerFactory.getZKHelixManager(_helixClusterName, _instanceId, InstanceType.SPECTATOR, _zkServers); try { spectatorManager.connect();
public PinotZKChanger(String zkAddress, String clusterName) { this.clusterName = clusterName; helixAdmin = new ZKHelixAdmin(zkAddress); helixManager = HelixManagerFactory .getZKHelixManager(clusterName, "PinotNumReplicaChanger", InstanceType.ADMINISTRATOR, zkAddress); try { helixManager.connect(); } catch (Exception e) { throw new RuntimeException(e); } ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName); propertyStore = new ZkHelixPropertyStore<>(zkAddress, serializer, path); }
.getZKHelixManager(this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);
@Test void testJobShouldGetCancelled() throws Exception { this.suite =new IntegrationJobCancelSuite(); Config helixConfig = this.suite.getManagerConfig(); String clusterName = helixConfig.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); String instanceName = ConfigUtils.getString(helixConfig, GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, GobblinClusterManager.class.getSimpleName()); String zkConnectString = helixConfig.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); HelixManager helixManager = HelixManagerFactory.getZKHelixManager(clusterName, instanceName, InstanceType.CONTROLLER, zkConnectString); suite.startCluster(); helixManager.connect(); TaskDriver taskDriver = new TaskDriver(helixManager); while (TaskDriver.getWorkflowContext(helixManager, IntegrationJobCancelSuite.JOB_ID) == null) { log.warn("Waiting for the job to start..."); Thread.sleep(1000L); } // Give the job some time to reach writer, where it sleeps Thread.sleep(2000L); log.info("Stopping the job"); taskDriver.stop(IntegrationJobCancelSuite.JOB_ID); suite.shutdownCluster(); suite.waitForAndVerifyOutputFiles(); }
this.helixManager = HelixManagerFactory.getZKHelixManager( helixClusterName, TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.SPECTATOR, zkConnectingString); this.helixManager.connect();
public static void addFakeDataInstanceToAutoJoinHelixCluster(String helixClusterName, String zkServer, String instanceId, boolean isSingleTenant, int adminPort) throws Exception { final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, instanceId, InstanceType.PARTICIPANT, zkServer); final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine(); final StateModelFactory<?> stateModelFactory = new EmptySegmentOnlineOfflineStateModelFactory(); stateMachineEngine .registerStateModelFactory(EmptySegmentOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); helixZkManager.connect(); if (isSingleTenant) { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.OFFLINE.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); } else { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, UNTAGGED_SERVER_INSTANCE); } HelixConfigScope scope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.PARTICIPANT, helixClusterName) .forParticipant(instanceId).build(); Map<String, String> props = new HashMap<>(); props.put(CommonConstants.Helix.Instance.ADMIN_PORT_KEY, String.valueOf(adminPort)); helixZkManager.getClusterManagmentTool().setConfig(scope, props); }
@BeforeClass public void setUp() throws Exception { // Use a random ZK port this.testingZKServer = new TestingServer(-1); LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString()); URL url = GobblinClusterManagerTest.class.getClassLoader().getResource( GobblinClusterManager.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); Config config = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())) .resolve(); String zkConnectionString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY)); this.helixManager = HelixManagerFactory .getZKHelixManager(config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.PARTICIPANT, zkConnectionString); this.helixManager.connect(); this.helixManager.getMessagingService().registerMessageHandlerFactory(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, new TestShutdownMessageHandlerFactory(this)); this.gobblinClusterManager = new GobblinClusterManager(TestHelper.TEST_APPLICATION_NAME, TestHelper.TEST_APPLICATION_ID, config, Optional.<Path>absent()); this.gobblinClusterManager.getEventBus().register(this.gobblinClusterManager); this.gobblinClusterManager.connectHelixManager(); }
this.helixManager = HelixManagerFactory.getZKHelixManager(this.helixClusterName, GobblinClusterUtils.getHostname(), InstanceType.SPECTATOR, this.zkConnectionString);
.getZKHelixManager(helixClusterName, TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectingString); this.closer.register(new Closeable() {
public static void addFakeBrokerInstancesToAutoJoinHelixCluster(String helixClusterName, String zkServer, int numInstances, boolean isSingleTenant) throws Exception { for (int i = 0; i < numInstances; ++i) { final String brokerId = "Broker_localhost_" + i; final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, brokerId, InstanceType.PARTICIPANT, zkServer); final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine(); final StateModelFactory<?> stateModelFactory = new EmptyBrokerOnlineOfflineStateModelFactory(); stateMachineEngine .registerStateModelFactory(EmptyBrokerOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); helixZkManager.connect(); if (isSingleTenant) { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)); } else { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, UNTAGGED_BROKER_INSTANCE); } } }
this.helixManager = HelixManagerFactory.getZKHelixManager( this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);
HelixManagerFactory.getZKHelixManager(helixClusterName, brokerId, InstanceType.SPECTATOR, zkServers); _spectatorHelixManager.connect(); _helixAdmin = _spectatorHelixManager.getClusterManagmentTool(); HelixManagerFactory.getZKHelixManager(helixClusterName, brokerId, InstanceType.PARTICIPANT, zkServers); StateMachineEngine stateMachineEngine = _helixManager.getStateMachineEngine(); StateModelFactory<?> stateModelFactory =
HelixManagerFactory.getZKHelixManager(helixClusterName, _instanceId, InstanceType.PARTICIPANT, _zkServers); final StateMachineEngine stateMachineEngine = _helixManager.getStateMachineEngine(); _helixManager.connect();
/** * Helper method to start a fake server that only implements Helix part. * * @throws Exception */ private void startFakeServer() throws Exception { _serverInstance = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + NetUtil.getHostAddress() + "_" + CommonConstants.Helix.DEFAULT_SERVER_NETTY_PORT; // Create server instance with the fake server state model _serverHelixManager = HelixManagerFactory .getZKHelixManager(_clusterName, _serverInstance, InstanceType.PARTICIPANT, ZkStarter.DEFAULT_ZK_STR); _serverHelixManager.getStateMachineEngine() .registerStateModelFactory(SegmentOnlineOfflineStateModelFactory.getStateModelName(), new FakeServerSegmentStateModelFactory()); _serverHelixManager.connect(); // Add Helix tag to the server _serverHelixManager.getClusterManagmentTool().addInstanceTag(_clusterName, _serverInstance, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); // Initialize controller leader locator ControllerLeaderLocator.create(_serverHelixManager); }
public void start() throws Exception { _helixZkManager = HelixManagerFactory.getZKHelixManager(_helixClusterName, _instanceId, InstanceType.PARTICIPANT, _zkServer); StateMachineEngine stateMachineEngine = _helixZkManager.getStateMachineEngine(); // create a stateModelFactory that returns a statemodel object for each partition. TestOnlineOfflineStateModelFactory stateModelFactory = new TestOnlineOfflineStateModelFactory(_instanceId, 0); stateMachineEngine.registerStateModelFactory("OnlineOffline", stateModelFactory); _helixZkManager.connect(); }
/** * Build the {@link HelixManager} for the Application Master. */ private HelixManager buildHelixManager(Config config, String zkConnectionString) { String helixInstanceName = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, GobblinClusterManager.class.getSimpleName()); return HelixManagerFactory.getZKHelixManager( config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), helixInstanceName, InstanceType.CONTROLLER, zkConnectionString); }