public JobScheduler(Properties properties, SchedulerService scheduler) throws Exception { this.properties = properties; this.scheduler = scheduler; this.jobExecutor = Executors.newFixedThreadPool(Integer.parseInt( properties.getProperty(ConfigurationKeys.JOB_EXECUTOR_THREAD_POOL_SIZE_KEY, Integer.toString(ConfigurationKeys.DEFAULT_JOB_EXECUTOR_THREAD_POOL_SIZE))), ExecutorsUtils.newThreadFactory(Optional.of(LOG), Optional.of("JobScheduler-%d"))); this.jobConfigFileExtensions = Sets.newHashSet(Splitter.on(",") .omitEmptyStrings() .split(this.properties.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_EXTENSIONS_KEY, ConfigurationKeys.DEFAULT_JOB_CONFIG_FILE_EXTENSIONS))); long pollingInterval = Long.parseLong( this.properties.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL_KEY, Long.toString(ConfigurationKeys.DEFAULT_JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL))); this.pathAlterationDetector = new PathAlterationObserverScheduler(pollingInterval); this.waitForJobCompletion = Boolean.parseBoolean( this.properties.getProperty(ConfigurationKeys.SCHEDULER_WAIT_FOR_JOB_COMPLETION_KEY, ConfigurationKeys.DEFAULT_SCHEDULER_WAIT_FOR_JOB_COMPLETION)); if (this.properties.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)) { this.jobConfigFileDirPath = new Path(this.properties.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); this.listener = new PathAlterationListenerAdaptorForMonitor(jobConfigFileDirPath, this); } else { // This is needed because HelixJobScheduler does not use the same way of finding changed paths this.jobConfigFileDirPath = null; this.listener = null; } }
public ImmutableFSJobCatalog(Config sysConfig, PathAlterationObserver observer, Optional<MetricContext> parentMetricContext, boolean instrumentationEnabled) throws IOException { super(Optional.of(LOGGER), parentMetricContext, instrumentationEnabled, Optional.of(sysConfig)); this.sysConfig = sysConfig; ConfigAccessor cfgAccessor = new ConfigAccessor(this.sysConfig); this.jobConfDirPath = cfgAccessor.getJobConfDirPath(); this.fs = cfgAccessor.getJobConfDirFileSystem(); this.loader = new PullFileLoader(jobConfDirPath, jobConfDirPath.getFileSystem(new Configuration()), cfgAccessor.getJobConfigurationFileExtensions(), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); this.converter = new ImmutableFSJobCatalog.JobSpecConverter(this.jobConfDirPath, getInjectedExtension()); long pollingInterval = cfgAccessor.getPollingInterval(); if (pollingInterval == ConfigurationKeys.DISABLED_JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL) { this.pathAlterationDetector = null; } else { this.pathAlterationDetector = new PathAlterationObserverScheduler(pollingInterval); // If absent, the Optional object will be created automatically by addPathAlterationObserver Optional<PathAlterationObserver> observerOptional = Optional.fromNullable(observer); FSPathAlterationListenerAdaptor configFilelistener = new FSPathAlterationListenerAdaptor(this.jobConfDirPath, this.loader, this.sysConfig, this.listeners, this.converter); this.pathAlterationDetector.addPathAlterationObserver(configFilelistener, observerOptional, this.jobConfDirPath); } }
@Test(dependsOnMethods = {"testLoadJobConfigsForCommonPropsFile", "testloadGenericJobConfig"}) public void testPathAlterationObserver() throws Exception { PathAlterationObserverScheduler monitor = new PathAlterationObserverScheduler(1000); final Set<Path> fileAltered = Sets.newHashSet(); final Semaphore semaphore = new Semaphore(0);
@Test(enabled=false, dependsOnMethods = {"testloadGenericJobConfig"}) public void testPathAlterationObserver() throws Exception { PathAlterationObserverScheduler detector = new PathAlterationObserverScheduler(1000); final Set<Path> fileAltered = Sets.newHashSet(); final Semaphore semaphore = new Semaphore(0);
public JobScheduler(Properties properties, SchedulerService scheduler) throws Exception { this.properties = properties; this.scheduler = scheduler; this.jobExecutor = Executors.newFixedThreadPool(Integer.parseInt( properties.getProperty(ConfigurationKeys.JOB_EXECUTOR_THREAD_POOL_SIZE_KEY, Integer.toString(ConfigurationKeys.DEFAULT_JOB_EXECUTOR_THREAD_POOL_SIZE))), ExecutorsUtils.newThreadFactory(Optional.of(LOG), Optional.of("JobScheduler-%d"))); this.jobConfigFileExtensions = Sets.newHashSet(Splitter.on(",") .omitEmptyStrings() .split(this.properties.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_EXTENSIONS_KEY, ConfigurationKeys.DEFAULT_JOB_CONFIG_FILE_EXTENSIONS))); long pollingInterval = Long.parseLong( this.properties.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL_KEY, Long.toString(ConfigurationKeys.DEFAULT_JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL))); this.pathAlterationDetector = new PathAlterationObserverScheduler(pollingInterval); this.waitForJobCompletion = Boolean.parseBoolean( this.properties.getProperty(ConfigurationKeys.SCHEDULER_WAIT_FOR_JOB_COMPLETION_KEY, ConfigurationKeys.DEFAULT_SCHEDULER_WAIT_FOR_JOB_COMPLETION)); if (this.properties.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)) { this.jobConfigFileDirPath = new Path(this.properties.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); this.listener = new PathAlterationListenerAdaptorForMonitor(jobConfigFileDirPath, this); } else { // This is needed because HelixJobScheduler does not use the same way of finding changed paths this.jobConfigFileDirPath = null; this.listener = null; } }
public ImmutableFSJobCatalog(Config sysConfig, PathAlterationObserver observer, Optional<MetricContext> parentMetricContext, boolean instrumentationEnabled) throws IOException { super(Optional.of(LOGGER), parentMetricContext, instrumentationEnabled, Optional.of(sysConfig)); this.sysConfig = sysConfig; ConfigAccessor cfgAccessor = new ConfigAccessor(this.sysConfig); this.jobConfDirPath = cfgAccessor.getJobConfDirPath(); this.fs = cfgAccessor.getJobConfDirFileSystem(); this.loader = new PullFileLoader(jobConfDirPath, jobConfDirPath.getFileSystem(new Configuration()), cfgAccessor.getJobConfigurationFileExtensions(), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); this.converter = new ImmutableFSJobCatalog.JobSpecConverter(this.jobConfDirPath, getInjectedExtension()); long pollingInterval = cfgAccessor.getPollingInterval(); if (pollingInterval == ConfigurationKeys.DISABLED_JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL) { this.pathAlterationDetector = null; } else { this.pathAlterationDetector = new PathAlterationObserverScheduler(pollingInterval); // If absent, the Optional object will be created automatically by addPathAlterationObserver Optional<PathAlterationObserver> observerOptional = Optional.fromNullable(observer); FSPathAlterationListenerAdaptor configFilelistener = new FSPathAlterationListenerAdaptor(this.jobConfDirPath, this.loader, this.sysConfig, this.listeners, this.converter); this.pathAlterationDetector.addPathAlterationObserver(configFilelistener, observerOptional, this.jobConfDirPath); } }