private JobConfigurationManager create(Config config) { try { if (config.hasPath(GobblinClusterConfigurationKeys.JOB_CONFIGURATION_MANAGER_KEY)) { return (JobConfigurationManager) GobblinConstructorUtils.invokeFirstConstructor(Class.forName( config.getString(GobblinClusterConfigurationKeys.JOB_CONFIGURATION_MANAGER_KEY)), ImmutableList.<Object>of(this.eventBus, config, this.jobCatalog), ImmutableList.<Object>of(this.eventBus, config)); } else { return new JobConfigurationManager(this.eventBus, config); } } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException(e); } }
@Override public Collection<StandardMetrics> getStandardMetricsCollection() { List<StandardMetrics> list = new ArrayList(); list.addAll(this.jobScheduler.getStandardMetricsCollection()); list.addAll(this.multiManager.getStandardMetricsCollection()); list.addAll(this.jobCatalog.getStandardMetricsCollection()); list.addAll(this.jobConfigurationManager.getStandardMetricsCollection()); return list; }
@Override protected void startUp() throws Exception { if (this.jobConfDirPath.isPresent()) { File path = new File(this.jobConfDirPath.get()); File jobConfigDir = path; // Backward compatibility: Previous impl was forcing users to look for jobConf within ${user.dir} // .. so if jobConfigDir does not exists, try to resolve config path via legacy route for backward // .. compatibility if (!path.exists()) { String pwd = System.getProperty("user.dir"); jobConfigDir = new File(pwd, path.getName() + GobblinClusterConfigurationKeys.TAR_GZ_FILE_SUFFIX); } if (jobConfigDir.exists()) { LOGGER.info("Loading job configurations from " + jobConfigDir); Properties properties = ConfigUtils.configToProperties(this.config); properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, "file://" + jobConfigDir.getAbsolutePath()); List<Properties> jobConfigs = SchedulerUtils.loadGenericJobConfigs(properties); LOGGER.info("Loaded " + jobConfigs.size() + " job configuration(s)"); for (Properties config : jobConfigs) { postNewJobConfigArrival(config.getProperty(ConfigurationKeys.JOB_NAME_KEY), config); } } else { LOGGER.warn("Job configuration directory " + jobConfigDir + " not found"); } } }
@BeforeClass public void setUp() throws IOException { this.eventBus.register(this); // Prepare the test job configuration files Assert.assertTrue(this.jobConfigFileDir.mkdirs(), "Failed to create " + this.jobConfigFileDir); Closer closer = Closer.create(); try { for (int i = 0; i < NUM_JOB_CONFIG_FILES; i++) { File jobConfigFile = new File(this.jobConfigFileDir, "test" + i + ".job"); Assert.assertTrue(jobConfigFile.createNewFile()); Properties properties = new Properties(); properties.setProperty("foo", "bar" + i); properties.store(closer.register(Files.newWriter(jobConfigFile, ConfigurationKeys.DEFAULT_CHARSET_ENCODING)), ""); } } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } Config config = ConfigFactory.empty().withValue(GobblinClusterConfigurationKeys.JOB_CONF_PATH_KEY, ConfigValueFactory.fromAnyRef(JOB_CONFIG_DIR_NAME)); this.jobConfigurationManager = new JobConfigurationManager(this.eventBus, config); this.jobConfigurationManager.startAsync().awaitRunning(); }
@AfterClass public void tearDown() throws IOException { this.jobConfigurationManager.stopAsync().awaitTerminated(); if (this.jobConfigFileDir.exists()) { FileUtils.deleteDirectory(this.jobConfigFileDir); } }
private JobConfigurationManager create(Config config) { try { if (config.hasPath(GobblinClusterConfigurationKeys.JOB_CONFIGURATION_MANAGER_KEY)) { return (JobConfigurationManager) GobblinConstructorUtils.invokeFirstConstructor(Class.forName( config.getString(GobblinClusterConfigurationKeys.JOB_CONFIGURATION_MANAGER_KEY)), ImmutableList.<Object>of(this.eventBus, config, this.jobCatalog), ImmutableList.<Object>of(this.eventBus, config)); } else { return new JobConfigurationManager(this.eventBus, config); } } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException(e); } }
@Override public Collection<StandardMetrics> getStandardMetricsCollection() { List<StandardMetrics> list = new ArrayList(); list.addAll(this.jobScheduler.getStandardMetricsCollection()); list.addAll(this.multiManager.getStandardMetricsCollection()); list.addAll(this.jobCatalog.getStandardMetricsCollection()); list.addAll(this.jobConfigurationManager.getStandardMetricsCollection()); return list; }
@Override protected void startUp() throws Exception { if (this.jobConfDirPath.isPresent()) { File path = new File(this.jobConfDirPath.get()); File jobConfigDir = path; // Backward compatibility: Previous impl was forcing users to look for jobConf within ${user.dir} // .. so if jobConfigDir does not exists, try to resolve config path via legacy route for backward // .. compatibility if (!path.exists()) { String pwd = System.getProperty("user.dir"); jobConfigDir = new File(pwd, path.getName() + GobblinClusterConfigurationKeys.TAR_GZ_FILE_SUFFIX); } if (jobConfigDir.exists()) { LOGGER.info("Loading job configurations from " + jobConfigDir); Properties properties = ConfigUtils.configToProperties(this.config); properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, "file://" + jobConfigDir.getAbsolutePath()); List<Properties> jobConfigs = SchedulerUtils.loadGenericJobConfigs(properties); LOGGER.info("Loaded " + jobConfigs.size() + " job configuration(s)"); for (Properties config : jobConfigs) { postNewJobConfigArrival(config.getProperty(ConfigurationKeys.JOB_NAME_KEY), config); } } else { LOGGER.warn("Job configuration directory " + jobConfigDir + " not found"); } } }