/** * Convert a given {@link Config} instance to a {@link Properties} instance. * * @param config the given {@link Config} instance * @return a {@link Properties} instance */ public static Properties configToProperties(Config config) { return configToProperties(config, Optional.<String>absent()); }
/** * Convert a given {@link Config} instance to a {@link Properties} instance. * * @param config the given {@link Config} instance * @param prefix only properties whose name starts with the prefix will be returned. * @return a {@link Properties} instance */ public static Properties configToProperties(Config config, String prefix) { return configToProperties(config, Optional.of(prefix)); }
public Properties getConfigAsProperties() { if (!this.configAsProperties.isPresent()) { this.configAsProperties = Optional.of(ConfigUtils.configToProperties(this.config.get())); } return this.configAsProperties.get(); }
public Properties getConfigAsProperties() { if (!this.configAsProperties.isPresent()) { this.configAsProperties = Optional.of(ConfigUtils.configToProperties(this.config.get())); } return this.configAsProperties.get(); }
public SchedulerService(Config cfg) { this(cfg.hasPath(ConfigurationKeys.SCHEDULER_WAIT_FOR_JOB_COMPLETION_KEY) ? cfg.getBoolean(ConfigurationKeys.SCHEDULER_WAIT_FOR_JOB_COMPLETION_KEY) : Boolean.parseBoolean(ConfigurationKeys.DEFAULT_SCHEDULER_WAIT_FOR_JOB_COMPLETION), Optional.of(ConfigUtils.configToProperties(cfg, "org.quartz."))); }
public Properties getConfigAsProperties() { if (!this.configAsProperties.isPresent()) { this.configAsProperties = Optional.of(ConfigUtils.configToProperties(this.config.get())); } return this.configAsProperties.get(); }
public GobblinServiceJobScheduler(Config config, Optional<HelixManager> helixManager, Optional<FlowCatalog> flowCatalog, Optional<TopologyCatalog> topologyCatalog, Orchestrator orchestrator, SchedulerService schedulerService, Optional<Logger> log) throws Exception { super(ConfigUtils.configToProperties(config), schedulerService); _log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass()); this.flowCatalog = flowCatalog; this.helixManager = helixManager; this.orchestrator = orchestrator; this.scheduledFlowSpecs = Maps.newHashMap(); }
/** * Convert a given {@link Config} to a {@link State} instance. * * @param config the given {@link Config} instance * @return a {@link State} instance */ public static State configToState(Config config) { return new State(configToProperties(config)); }
/** * Build the {@link GobblinHelixJobScheduler} for the Application Master. */ private GobblinHelixJobScheduler buildGobblinHelixJobScheduler(Config config, Path appWorkDir, List<? extends Tag<?>> metadataTags, SchedulerService schedulerService) throws Exception { Properties properties = ConfigUtils.configToProperties(config); return new GobblinHelixJobScheduler(properties, this.helixManager, this.eventBus, appWorkDir, metadataTags, schedulerService, this.jobCatalog); }
@Override public HivePartitionExtendedFilter createFilter(Config config){ Properties props = ConfigUtils.configToProperties(config); return props.containsKey(PathBasedHivePartitionFilterFactory.HIVE_PARTITION_PATH_FILTER_REGEX)? new PathBasedPartitionFilter(props.getProperty(PathBasedHivePartitionFilterFactory.HIVE_PARTITION_PATH_FILTER_REGEX)) :null; } }
protected ConsumerConfig createConsumerConfig(Config config) { Properties props = ConfigUtils.configToProperties(config); if (!props.containsKey(GROUP_ID_KEY)) { props.setProperty(GROUP_ID_KEY, DEFAULT_GROUP_ID); } return new ConsumerConfig(props); }
private Kafka08DataWriter<byte[]> getKafka08Producer() { if (null == _kafka08Producer) { _kafka08Producer = new Kafka08DataWriter<byte[]>(ConfigUtils.configToProperties(_config)); } return _kafka08Producer; }
public static DefaultConfigurableImpl createFromConfig(Config srcConfig) { return new DefaultConfigurableImpl(srcConfig, ConfigUtils.configToProperties(srcConfig)); }
@SuppressWarnings("unchecked") protected VersionSelectionPolicy<DatasetVersion> createSelectionPolicy(Config selectionConfig, Config jobConfig) { try { String selectionPolicyKey = StringUtils.substringAfter(ConfigurableCleanableDataset.SELECTION_POLICY_CLASS_KEY, ConfigurableCleanableDataset.CONFIGURATION_KEY_PREFIX); Preconditions.checkArgument(selectionConfig.hasPath(selectionPolicyKey)); String className = selectionConfig.getString(selectionPolicyKey); return (VersionSelectionPolicy<DatasetVersion>) GobblinConstructorUtils.invokeFirstConstructor( this.versionSelectionAliasResolver.resolveClass(className), ImmutableList.<Object> of(selectionConfig), ImmutableList.<Object> of(selectionConfig, ConfigUtils.configToProperties(jobConfig)), ImmutableList.<Object> of(ConfigUtils.configToProperties(jobConfig))); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new IllegalArgumentException(e); } } }
/** * Called once on {@link #startUp()} to start metrics. */ @VisibleForTesting protected void buildMetricsContextAndMetrics() { this.metricContext = Instrumented.getMetricContext(new gobblin.configuration.State(ConfigUtils.configToProperties(config)), this.getClass(), getTagsForMetrics()); createMetrics(); }
private static Properties resolveTemplate(Properties jobProps) throws IOException { try { if (jobProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) { Config jobConfig = ConfigUtils.propertiesToConfig(jobProps); Properties resolvedProps = ConfigUtils.configToProperties((ResourceBasedJobTemplate .forResourcePath(jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH), new PackagedTemplatesJobCatalogDecorator())) .getResolvedConfig(jobConfig)); return resolvedProps; } else { return jobProps; } } catch (JobTemplate.TemplateException | SpecNotFoundException | URISyntaxException exc) { throw new IOException(exc); } } }
/** * Load job configuration from job configuration files stored in general file system, * located by Path * @param sysProps Gobblin framework configuration properties * @return a list of job configurations in the form of {@link java.util.Properties} */ public static List<Properties> loadGenericJobConfigs(Properties sysProps) throws ConfigurationException, IOException { Path rootPath = new Path(sysProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); PullFileLoader loader = new PullFileLoader(rootPath, rootPath.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(rootPath, sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config))); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; }
/** * Load a given job configuration file from a general file system. * * @param sysProps Gobblin framework configuration properties * @param jobConfigPath job configuration file to be loaded * @param jobConfigPathDir root job configuration file directory * @return a job configuration in the form of {@link java.util.Properties} */ public static Properties loadGenericJobConfig(Properties sysProps, Path jobConfigPath, Path jobConfigPathDir) throws ConfigurationException, IOException { PullFileLoader loader = new PullFileLoader(jobConfigPathDir, jobConfigPathDir.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Config config = loader.loadPullFile(jobConfigPath, sysConfig, true); return resolveTemplate(ConfigUtils.configToProperties(config)); }
public GobblinTaskRunner(String applicationName, String helixInstanceName, String applicationId, String taskRunnerId, Config config, Optional<Path> appWorkDirOptional) throws Exception { this.helixInstanceName = helixInstanceName; this.config = config; this.taskRunnerId = taskRunnerId; Configuration conf = HadoopUtils.newConfiguration(); this.fs = buildFileSystem(this.config, conf); String zkConnectionString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); LOGGER.info("Using ZooKeeper connection string: " + zkConnectionString); this.helixManager = HelixManagerFactory .getZKHelixManager(config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), helixInstanceName, InstanceType.PARTICIPANT, zkConnectionString); Properties properties = ConfigUtils.configToProperties(config); TaskExecutor taskExecutor = new TaskExecutor(properties); TaskStateTracker taskStateTracker = new GobblinHelixTaskStateTracker(properties, this.helixManager); Path appWorkDir = appWorkDirOptional.isPresent() ? appWorkDirOptional.get() : GobblinClusterUtils.getAppWorkDirPath(this.fs, applicationName, applicationId); List<Service> services = Lists.newArrayList(taskExecutor, taskStateTracker, new JMXReportingService()); services.addAll(getServices()); this.serviceManager = new ServiceManager(services); this.containerMetrics = buildContainerMetrics(this.config, properties, applicationName, this.taskRunnerId); // Register task factory for the Helix task state model Map<String, TaskFactory> taskFactoryMap = Maps.newHashMap(); taskFactoryMap.put(GOBBLIN_TASK_FACTORY_NAME, new GobblinHelixTaskFactory(this.containerMetrics, taskExecutor, taskStateTracker, this.fs, appWorkDir, config)); this.taskStateModelFactory = new TaskStateModelFactory(this.helixManager, taskFactoryMap); this.helixManager.getStateMachineEngine().registerStateModelFactory("Task", this.taskStateModelFactory); }
/** * Resolve the job spec using classpath templates as well as any templates available in the input {@link JobCatalog}. */ public ResolvedJobSpec(JobSpec other, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { super(other.getUri(), other.getVersion(), other.getDescription(), resolveConfig(other, catalog), ConfigUtils.configToProperties(resolveConfig(other, catalog)), other.getTemplateURI()); this.originalJobSpec = other; }