/** * Create a Helix cluster for the Gobblin Cluster application. * * @param zkConnectionString the ZooKeeper connection string * @param clusterName the Helix cluster name */ public static void createGobblinHelixCluster( String zkConnectionString, String clusterName) { createGobblinHelixCluster(zkConnectionString, clusterName, true); }
/** * Launch a new Gobblin cluster on AWS. * * @throws IOException If there's something wrong launching the cluster */ public void launch() throws IOException, InterruptedException { this.eventBus.register(this); // Create Helix cluster and connect to it HelixUtils.createGobblinHelixCluster(this.zkConnectionString, this.helixClusterName, false); LOGGER.info("Created Helix cluster " + this.helixClusterName); connectHelixManager(); // Core logic to launch cluster this.clusterId = getClusterId(); // TODO: Add cluster monitoring countDownLatch.await(); }
protected void createHelixCluster() throws Exception { String zkConnectionString = this.managerConfig .getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); String helix_cluster_name = this.managerConfig .getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, helix_cluster_name); } }
@Override public void createHelixCluster() throws Exception { super.createHelixCluster(); String zkConnectionString = managerConfig .getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); String managerClusterName = managerConfig .getString(GobblinClusterConfigurationKeys.MANAGER_CLUSTER_NAME_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, managerClusterName); String taskDriverClusterName = taskDriverConfigs.iterator().next() .getString(GobblinClusterConfigurationKeys.TASK_DRIVER_CLUSTER_NAME_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, taskDriverClusterName); }
@Override public void createHelixCluster() throws Exception { super.createHelixCluster(); String zkConnectionString = managerConfig .getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); String manager_cluster_name = managerConfig .getString(GobblinClusterConfigurationKeys.MANAGER_CLUSTER_NAME_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, manager_cluster_name); }
@Test public void testCreateHelixCluster() throws Exception { // This is tested here instead of in HelixUtilsTest to avoid setting up yet another testing ZooKeeper server. HelixUtils .createGobblinHelixCluster(this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY), this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY)); // Assert to check if there is no pre-existing cluster Assert.assertEquals(this.curatorFramework.checkExists().forPath(String.format("/%s", this.helixClusterName)).getVersion(), 0); Assert.assertEquals(this.curatorFramework.checkExists().forPath(String.format("/%s/CONTROLLER", this.helixClusterName)).getVersion(), 0); }
@Test public void testCreateHelixCluster() throws Exception { // This is tested here instead of in HelixUtilsTest to avoid setting up yet another testing ZooKeeper server. HelixUtils.createGobblinHelixCluster( this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY), this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY)); Assert.assertEquals(this.curatorFramework.checkExists() .forPath(String.format("/%s", GobblinYarnAppLauncherTest.class.getSimpleName())).getVersion(), 0); Assert.assertEquals( this.curatorFramework.checkExists() .forPath(String.format("/%s/CONTROLLER", GobblinYarnAppLauncherTest.class.getSimpleName())).getVersion(), 0); }
HelixUtils.createGobblinHelixCluster( this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY), clusterName); LOGGER.info("Created Helix cluster " + clusterName);
HelixUtils.createGobblinHelixCluster(zkConnectionString, helixClusterName, false); LOGGER.info("Created Helix cluster " + helixClusterName);
@BeforeClass public void setUp() throws Exception { this.testingZKServer = new TestingServer(-1); LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString()); URL url = GobblinTaskRunnerTest.class.getClassLoader().getResource( GobblinTaskRunnerTest.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); Config config = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())) .resolve(); String zkConnectionString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY)); // Participant this.gobblinTaskRunner = new GobblinTaskRunner(TestHelper.TEST_APPLICATION_NAME, TestHelper.TEST_HELIX_INSTANCE_NAME, TestHelper.TEST_APPLICATION_ID, TestHelper.TEST_TASK_RUNNER_ID, config, Optional.<Path>absent()); this.gobblinTaskRunner.connectHelixManager(); // Controller this.gobblinClusterManager = new GobblinClusterManager(TestHelper.TEST_APPLICATION_NAME, TestHelper.TEST_APPLICATION_ID, config, Optional.<Path>absent()); this.gobblinClusterManager.connectHelixManager(); }
String helixClusterName = config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); HelixUtils.createGobblinHelixCluster(zkConnectingString, helixClusterName);
@BeforeClass public void setUp() throws Exception { // Use a random ZK port this.testingZKServer = new TestingServer(-1); LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString()); URL url = GobblinClusterManagerTest.class.getClassLoader().getResource( GobblinClusterManager.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); Config config = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())) .resolve(); String zkConnectionString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY)); this.helixManager = HelixManagerFactory .getZKHelixManager(config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.PARTICIPANT, zkConnectionString); this.helixManager.connect(); this.helixManager.getMessagingService().registerMessageHandlerFactory(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, new TestShutdownMessageHandlerFactory(this)); this.gobblinClusterManager = new GobblinClusterManager(TestHelper.TEST_APPLICATION_NAME, TestHelper.TEST_APPLICATION_ID, config, Optional.<Path>absent()); this.gobblinClusterManager.getEventBus().register(this.gobblinClusterManager); this.gobblinClusterManager.connectHelixManager(); }
@BeforeMethod public void setUp() throws Exception { // Use a random ZK port _testingZKServer = new TestingServer(-1); LOG.info("Testing ZK Server listening on: " + _testingZKServer.getConnectString()); URL url = GobblinClusterKillTest.class.getClassLoader().getResource( GobblinClusterKillTest.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); _config = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(_testingZKServer.getConnectString())) .withValue("gobblin.cluster.jobconf.fullyQualifiedPath", ConfigValueFactory.fromAnyRef("/tmp/gobblinClusterKillTest/job-conf")) .resolve(); String zkConnectionString = _config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, _config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY)); setupTestDir(); _clusterManagers = new GobblinClusterManager[NUM_MANAGERS]; _clusterWorkers = new GobblinTaskRunner[NUM_WORKERS]; _workerStartThreads = new Thread[NUM_WORKERS]; for (int i = 0; i < NUM_MANAGERS; i++) { setupManager(i); } for (int i = 0; i < NUM_WORKERS; i++) { setupWorker(i); } }
String helixClusterName = baseConfig.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); HelixUtils.createGobblinHelixCluster(zkConnectingString, helixClusterName);
/** * Create a Helix cluster for the Gobblin Cluster application. * * @param zkConnectionString the ZooKeeper connection string * @param clusterName the Helix cluster name */ public static void createGobblinHelixCluster( String zkConnectionString, String clusterName) { createGobblinHelixCluster(zkConnectionString, clusterName, true); }
/** * Launch a new Gobblin cluster on AWS. * * @throws IOException If there's something wrong launching the cluster */ public void launch() throws IOException, InterruptedException { this.eventBus.register(this); // Create Helix cluster and connect to it HelixUtils.createGobblinHelixCluster(this.zkConnectionString, this.helixClusterName, false); LOGGER.info("Created Helix cluster " + this.helixClusterName); connectHelixManager(); // Core logic to launch cluster this.clusterId = getClusterId(); // TODO: Add cluster monitoring countDownLatch.await(); }
HelixUtils.createGobblinHelixCluster( this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY), clusterName); LOGGER.info("Created Helix cluster " + clusterName);
HelixUtils.createGobblinHelixCluster(zkConnectionString, helixClusterName, false); LOGGER.info("Created Helix cluster " + helixClusterName);