public KafkaJobMonitor(String topic, MutableJobCatalog catalog, Config config) { super(topic, ConfigUtils.getConfigOrEmpty(config, KAFKA_JOB_MONITOR_PREFIX), 1); this.jobCatalog = catalog; try { this.datasetStateStore = DatasetStateStore.buildDatasetStateStore(config); } catch (Exception e) { log.warn("DatasetStateStore could not be created.", e); } }
public JobStateToJsonConverter(Properties props, String storeUrl, boolean keepConfig) throws IOException { Configuration conf = new Configuration(); JobConfigurationUtils.putPropertiesIntoConfiguration(props, conf); if (StringUtils.isNotBlank(storeUrl)) { props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeUrl); } this.keepConfig = keepConfig; this.jobStateStore = (StateStore) DatasetStateStore.buildDatasetStateStore(ConfigUtils.propertiesToConfig(props)); }
public DatasetStoreDatasetFinder(FileSystem fs, Properties props) throws IOException { this.config = ConfigFactory.parseProperties(props); this.store = DatasetStateStore.buildDatasetStateStore(this.config); this.predicate = buildPredicate(); }
@Override public void run(String[] args) throws Exception { CliObjectFactory<Command> factory = new ConstructorAndPublicMethodsCliObjectFactory<>(Command.class); Command command = factory.buildObject(args, 1, true, args[0]); FileSystem fs = FileSystem.get(new Configuration()); FSDataInputStream inputStream = fs.open(command.path); Config config = ConfigFactory.parseReader(new InputStreamReader(inputStream, Charset.defaultCharset())); Preconditions.checkNotNull(config.getObject(SOURCE_KEY)); Preconditions.checkNotNull(config.getObject(DESTINATION_KEY)); DatasetStateStore dstDatasetStateStore = DatasetStateStore.buildDatasetStateStore(config.getConfig(DESTINATION_KEY)); DatasetStateStore srcDatasetStateStore = DatasetStateStore.buildDatasetStateStore(config.getConfig(SOURCE_KEY)); Map<String, JobState.DatasetState> map; // if migrating state for all jobs then list the store names (job names) and copy the current jst files if (ConfigUtils.getBoolean(config, MIGRATE_ALL_JOBS, Boolean.valueOf(DEFAULT_MIGRATE_ALL_JOBS))) { List<String> jobNames = srcDatasetStateStore.getStoreNames(Predicates.alwaysTrue()); for (String jobName : jobNames) { migrateStateForJob(srcDatasetStateStore, dstDatasetStateStore, jobName, command.deleteSourceStateStore); } } else { Preconditions.checkNotNull(config.getString(JOB_NAME_KEY)); migrateStateForJob(srcDatasetStateStore, dstDatasetStateStore, config.getString(JOB_NAME_KEY), command.deleteSourceStateStore); } }
public KafkaJobMonitor(String topic, MutableJobCatalog catalog, Config config) { super(topic, ConfigUtils.getConfigOrEmpty(config, KAFKA_JOB_MONITOR_PREFIX), 1); this.jobCatalog = catalog; try { this.datasetStateStore = DatasetStateStore.buildDatasetStateStore(config); } catch (Exception e) { log.warn("DatasetStateStore could not be created.", e); } }
public JobStateToJsonConverter(Properties props, String storeUrl, boolean keepConfig) throws IOException { Configuration conf = new Configuration(); JobConfigurationUtils.putPropertiesIntoConfiguration(props, conf); if (StringUtils.isNotBlank(storeUrl)) { props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeUrl); } this.keepConfig = keepConfig; this.jobStateStore = (StateStore) DatasetStateStore.buildDatasetStateStore(ConfigUtils.propertiesToConfig(props)); }
public DatasetStoreDatasetFinder(FileSystem fs, Properties props) throws IOException { this.config = ConfigFactory.parseProperties(props); this.store = DatasetStateStore.buildDatasetStateStore(this.config); this.predicate = buildPredicate(); }
@Override public void run(String[] args) throws Exception { CliObjectFactory<Command> factory = new ConstructorAndPublicMethodsCliObjectFactory<>(Command.class); Command command = factory.buildObject(args, 1, true, args[0]); FileSystem fs = FileSystem.get(new Configuration()); FSDataInputStream inputStream = fs.open(command.path); Config config = ConfigFactory.parseReader(new InputStreamReader(inputStream, Charset.defaultCharset())); Preconditions.checkNotNull(config.getObject(SOURCE_KEY)); Preconditions.checkNotNull(config.getObject(DESTINATION_KEY)); DatasetStateStore dstDatasetStateStore = DatasetStateStore.buildDatasetStateStore(config.getConfig(DESTINATION_KEY)); DatasetStateStore srcDatasetStateStore = DatasetStateStore.buildDatasetStateStore(config.getConfig(SOURCE_KEY)); Map<String, JobState.DatasetState> map; // if migrating state for all jobs then list the store names (job names) and copy the current jst files if (ConfigUtils.getBoolean(config, MIGRATE_ALL_JOBS, Boolean.valueOf(DEFAULT_MIGRATE_ALL_JOBS))) { List<String> jobNames = srcDatasetStateStore.getStoreNames(Predicates.alwaysTrue()); for (String jobName : jobNames) { migrateStateForJob(srcDatasetStateStore, dstDatasetStateStore, jobName, command.deleteSourceStateStore); } } else { Preconditions.checkNotNull(config.getString(JOB_NAME_KEY)); migrateStateForJob(srcDatasetStateStore, dstDatasetStateStore, config.getString(JOB_NAME_KEY), command.deleteSourceStateStore); } }