public static void main(String[] args) throws Exception { Options options = buildOptions(); try { CommandLine cmd = new DefaultParser().parse(options, args); if (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) || !cmd .hasOption(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME)) { printUsage(options); System.exit(1); } logger.info(JvmUtils.getJvmInputArguments()); String applicationName = cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME); String helixInstanceName = cmd.getOptionValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME); GobblinTaskRunner gobblinWorkUnitRunner = new GobblinTaskRunner(applicationName, helixInstanceName, getApplicationId(), getTaskRunnerId(), ConfigFactory.load(), Optional.<Path>absent()); gobblinWorkUnitRunner.start(); } catch (ParseException pe) { printUsage(options); System.exit(1); } } }
/** * Start this {@link GobblinTaskRunner} instance. */ public void start() { logger.info(String.format("Starting %s in container %s", this.helixInstanceName, this.taskRunnerId)); // Add a shutdown hook so the task scheduler gets properly shutdown addShutdownHook(); connectHelixManager(); addInstanceTags(); // Start metric reporting if (this.containerMetrics.isPresent()) { this.containerMetrics.get() .startMetricReportingWithFileSuffix(ConfigUtils.configToState(this.config), this.taskRunnerId); } if (this.serviceManager != null) { this.serviceManager.startAsync(); this.serviceManager.awaitStopped(); } }
public synchronized void stop() { if (this.isStopped || this.stopInProgress) { return; } this.stopInProgress = true; logger.info("Stopping the Gobblin Task runner"); // Stop metric reporting if (this.containerMetrics.isPresent()) { this.containerMetrics.get().stopMetricsReporting(); } try { stopServices(); } finally { this.taskStateModelFactory.shutdown(); disconnectHelixManager(); } this.isStopped = true; }
GobblinClusterConfigurationKeys.DEDICATED_TASK_DRIVER_CLUSTER_ENABLED, false); Configuration conf = HadoopUtils.newConfiguration(); this.fs = buildFileSystem(config, conf); this.appWorkPath = initAppWorkDir(config, appWorkDirOptional); this.config = saveConfigToFile(config); this.clusterName = this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); initHelixManager(); this.containerMetrics = buildContainerMetrics(); .build(); this.taskStateModelFactory = createTaskStateModelFactory(suite.getTaskFactoryMap()); this.metricsCollection = suite.getMetricsCollection(); this.services.addAll(suite.getServices()); this.services.addAll(getServices()); if (this.services.isEmpty()) { this.serviceManager = null;
@Override public void run() { gobblinTaskRunner.start(); } });
@BeforeClass public void setUp() throws Exception { this.testingZKServer = new TestingServer(-1); LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString()); URL url = GobblinTaskRunnerTest.class.getClassLoader().getResource( GobblinTaskRunnerTest.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); Config config = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())) .resolve(); String zkConnectionString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY)); // Participant this.gobblinTaskRunner = new GobblinTaskRunner(TestHelper.TEST_APPLICATION_NAME, TestHelper.TEST_HELIX_INSTANCE_NAME, TestHelper.TEST_APPLICATION_ID, TestHelper.TEST_TASK_RUNNER_ID, config, Optional.<Path>absent()); this.gobblinTaskRunner.connectHelixManager(); // Controller this.gobblinClusterManager = new GobblinClusterManager(TestHelper.TEST_APPLICATION_NAME, TestHelper.TEST_APPLICATION_ID, config, Optional.<Path>absent()); this.gobblinClusterManager.connectHelixManager(); }
@Override public void run() { logger.info("Running the shutdown hook"); GobblinTaskRunner.this.stop(); } });
private void startTaskDriver() throws Exception { for (Config taskDriverConfig: this.taskDriverConfigs) { GobblinTaskRunner runner = new GobblinTaskRunner(TestHelper.TEST_APPLICATION_NAME, taskDriverConfig.getString(TEST_INSTANCE_NAME_KEY), TestHelper.TEST_APPLICATION_ID, "1", taskDriverConfig, Optional.absent()); this.taskDrivers.add(runner); // Need to run in another thread since the start call will not return until the stop method // is called. Thread workerThread = new Thread(runner::start); workerThread.start(); } }
@AfterClass public void tearDown() throws IOException { try { this.gobblinClusterManager.disconnectHelixManager(); this.gobblinTaskRunner.disconnectHelixManager(); } finally { this.testingZKServer.close(); } } }
this.fs = buildFileSystem(config, conf); this.appWorkPath = initAppWorkDir(config, appWorkDirOptional); this.config = saveConfigToFile(config); this.clusterName = this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); initHelixManager(); this.containerMetrics = buildContainerMetrics(); .build(); this.taskStateModelFactory = createTaskStateModelFactory(suite.getTaskFactoryMap()); this.metricsCollection = suite.getMetricsCollection(); this.services.addAll(suite.getServices()); this.services.addAll(getServices()); if (this.services.isEmpty()) { this.serviceManager = null;
@Override public void run() { fworker.start(); } });
@AfterClass public void tearDown() throws IOException { try { this.gobblinTaskRunner.stop(); this.thread.join(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } finally { this.closer.close(); } }
/** * Create and start a cluster worker * @param id - array offset * @throws Exception */ private void setupWorker(int id) throws Exception { final GobblinTaskRunner fworker = new GobblinTaskRunner(TestHelper.TEST_APPLICATION_NAME, "Worker_" + id, TestHelper.TEST_APPLICATION_ID, "1", _config, Optional.of(new Path(_config.getString("gobblin.cluster.work.dir")))); _clusterWorkers[id] = fworker; _workerStartThreads[id] = new Thread(new Runnable() { @Override public void run() { fworker.start(); } }); _workerStartThreads[id].start(); }
FileUtils.listFiles(writerOutputDir, new String[]{"txt"}, true).size()); _clusterWorkers[0].disconnectHelixManager();
public static void main(String[] args) throws Exception { Options options = buildOptions(); try { CommandLine cmd = new DefaultParser().parse(options, args); if (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) || !cmd .hasOption(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME)) { printUsage(options); System.exit(1); } logger.info(JvmUtils.getJvmInputArguments()); String applicationName = cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME); String helixInstanceName = cmd.getOptionValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME); GobblinTaskRunner gobblinWorkUnitRunner = new GobblinTaskRunner(applicationName, helixInstanceName, getApplicationId(), getTaskRunnerId(), ConfigFactory.load(), Optional.<Path>absent()); gobblinWorkUnitRunner.start(); } catch (ParseException pe) { printUsage(options); System.exit(1); } } }
new GobblinAWSTaskRunner(applicationName, helixInstanceName, ConfigFactory.load(), Optional.of(new Path(appWorkDir))); gobblinTaskRunner.start(); } catch (ParseException pe) { printUsage(options);
/** * Start this {@link GobblinTaskRunner} instance. */ public void start() { logger.info(String.format("Starting %s in container %s", this.helixInstanceName, this.taskRunnerId)); // Add a shutdown hook so the task scheduler gets properly shutdown addShutdownHook(); connectHelixManager(); addInstanceTags(); // Start metric reporting if (this.containerMetrics.isPresent()) { this.containerMetrics.get() .startMetricReportingWithFileSuffix(ConfigUtils.configToState(this.config), this.taskRunnerId); } if (this.serviceManager != null) { this.serviceManager.startAsync(); this.serviceManager.awaitStopped(); } }
public synchronized void stop() { if (this.isStopped || this.stopInProgress) { return; } this.stopInProgress = true; logger.info("Stopping the Gobblin Task runner"); // Stop metric reporting if (this.containerMetrics.isPresent()) { this.containerMetrics.get().stopMetricsReporting(); } try { stopServices(); } finally { this.taskStateModelFactory.shutdown(); disconnectHelixManager(); } this.isStopped = true; }
private void startWorker() throws Exception { if (workerConfigs.size() == 1) { this.workers.add(new GobblinTaskRunner(TestHelper.TEST_APPLICATION_NAME, WORKER_INSTANCE_0, TestHelper.TEST_APPLICATION_ID, "1", this.workerConfigs.iterator().next(), Optional.absent())); // Need to run in another thread since the start call will not return until the stop method // is called. Thread workerThread = new Thread(this.workers.iterator().next()::start); workerThread.start(); } else { // Each workerConfig corresponds to a worker instance for (Config workerConfig: this.workerConfigs) { GobblinTaskRunner runner = new GobblinTaskRunner(TestHelper.TEST_APPLICATION_NAME, workerConfig.getString(TEST_INSTANCE_NAME_KEY), TestHelper.TEST_APPLICATION_ID, "1", workerConfig, Optional.absent()); this.workers.add(runner); // Need to run in another thread since the start call will not return until the stop method // is called. Thread workerThread = new Thread(runner::start); workerThread.start(); } } }