private Boolean getIsRunTaskInSeparateProcessEnabled(Config config) { return ConfigUtils.getBoolean(config, GobblinClusterConfigurationKeys.ENABLE_TASK_IN_SEPARATE_PROCESS, false); } }
public MultiVersionCleanableDatasetBase(final FileSystem fs, final Properties props, Config config, Logger log) throws IOException { this(fs, props, Boolean.valueOf(props.getProperty(SIMULATE_KEY, SIMULATE_DEFAULT)), Boolean.valueOf(props.getProperty(SKIP_TRASH_KEY, SKIP_TRASH_DEFAULT)), Boolean.valueOf(props.getProperty(DELETE_EMPTY_DIRECTORIES_KEY, DELETE_EMPTY_DIRECTORIES_DEFAULT)), Boolean.valueOf(props.getProperty(DELETE_AS_OWNER_KEY, DELETE_AS_OWNER_DEFAULT)), ConfigUtils.getBoolean(config, IS_DATASET_BLACKLISTED_KEY, Boolean.valueOf(IS_DATASET_BLACKLISTED_DEFAULT)), log); }
ElasticsearchTransportClientWriter(Config config) throws UnknownHostException { super(config); // Check if ssl is being configured, throw error that transport client does not support ssl Preconditions.checkArgument(!ConfigUtils.getBoolean(config, ElasticsearchWriterConfigurationKeys.ELASTICSEARCH_WRITER_SSL_ENABLED, false), "Transport client does not support ssl, try the Rest client instead"); this.client = createTransportClient(config); log.info("ElasticsearchWriter configured successfully with: indexName={}, indexType={}, idMappingEnabled={}, typeMapperClassName={}", this.indexName, this.indexType, this.idMappingEnabled, this.typeMapper); }
public RetentionAction(Config actionConfig, FileSystem fs, Config jobConfig) { this.versionSelectionAliasResolver = new ClassAliasResolver<>(VersionSelectionPolicy.class); this.fs = fs; this.isSimulateMode = ConfigUtils.getBoolean(jobConfig, FsCleanableHelper.SIMULATE_KEY, Boolean.valueOf(FsCleanableHelper.SIMULATE_DEFAULT)); }
@Override public DatasetStateStore<JobState.DatasetState> createStateStore(Config config) { String connectString = config.getString(ZkStateStoreConfigurationKeys.STATE_STORE_ZK_CONNECT_STRING_KEY); String rootDir = config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { return new ZkDatasetStateStore(connectString, rootDir, compressedValues); } catch (Exception e) { throw new RuntimeException("Failed to create ZkDatasetStateStore with factory", e); } } }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { String connectString = ConfigUtils.getString(config, ZkStateStoreConfigurationKeys.STATE_STORE_ZK_CONNECT_STRING_KEY, ZkStateStoreConfigurationKeys.STATE_STORE_ZK_CONNECT_STRING_DEFAULT); String rootDir = config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { return new ZkStateStore(connectString, rootDir, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create ZkStateStore with factory", e); } } }
private ConversionConfig(Config config, Table table, String destinationFormat) { super(config, table); // Required this.destinationFormat = destinationFormat; // Optional this.destinationViewName = Optional.fromNullable(resolveTemplate(ConfigUtils.getString(config, DESTINATION_VIEW_KEY, null), table)); this.updateViewAlwaysEnabled = ConfigUtils.getBoolean(config, UPDATE_VIEW_ALWAYS_ENABLED, true); }
public GobblinHelixMultiManager( Config config, Function<Void, MessageHandlerFactory> messageHandlerFactoryFunction, EventBus eventBus, GobblinClusterManager.StopStatus stopStatus) { this.config = config; this.eventBus = eventBus; this.stopStatus = stopStatus; this.isStandaloneMode = ConfigUtils.getBoolean(config, GobblinClusterConfigurationKeys.STANDALONE_CLUSTER_MODE_KEY, GobblinClusterConfigurationKeys.DEFAULT_STANDALONE_CLUSTER_MODE); this.metricContext = Instrumented.getMetricContext(ConfigUtils.configToState(config), this.getClass()); this.metrics = new HelixManagerMetrics(this.metricContext, this.config); this.dedicatedManagerCluster = ConfigUtils.getBoolean(config, GobblinClusterConfigurationKeys.DEDICATED_MANAGER_CLUSTER_ENABLED,false); this.dedicatedTaskDriverCluster = ConfigUtils.getBoolean(config, GobblinClusterConfigurationKeys.DEDICATED_TASK_DRIVER_CLUSTER_ENABLED, false); this.userDefinedMessageHandlerFactory = messageHandlerFactoryFunction.apply(null); initialize(); }
public FSDatasetDescriptor(Config config) { Preconditions.checkArgument(config.hasPath(DatasetDescriptorConfigKeys.PLATFORM_KEY), "Dataset descriptor config must specify platform"); this.platform = config.getString(DatasetDescriptorConfigKeys.PLATFORM_KEY); this.path = PathUtils.getPathWithoutSchemeAndAuthority(new Path(ConfigUtils.getString(config, DatasetDescriptorConfigKeys.PATH_KEY, DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_ANY))).toString(); this.formatConfig = new FormatConfig(config); this.isRetentionApplied = ConfigUtils.getBoolean(config, DatasetDescriptorConfigKeys.IS_RETENTION_APPLIED_KEY, false); this.description = ConfigUtils.getString(config, DatasetDescriptorConfigKeys.DESCRIPTION_KEY, ""); this.rawConfig = config.withFallback(this.formatConfig.getRawConfig()).withFallback(DEFAULT_FALLBACK); }
private static Config addSystemConfigurationToConfig(Config config) { Map<String, String> confMap = Maps.newHashMap(); addBrokerKeys(confMap, System.getenv().entrySet()); addBrokerKeys(confMap, System.getProperties().entrySet()); Config systemConfig = ConfigFactory.parseMap(confMap); Config tmpConfig = config.withFallback(systemConfig); String brokerConfPath = DEFAULT_BROKER_CONF_FILE; if (tmpConfig.hasPath(BROKER_CONF_FILE_KEY)) { brokerConfPath = tmpConfig.getString(BROKER_CONF_FILE_KEY); } Config resourceConfig = ConfigFactory.parseResources(SharedResourcesBrokerFactory.class, brokerConfPath); config = config.withFallback(resourceConfig).withFallback(systemConfig); if (ConfigUtils.getBoolean(config, LOAD_HADOOP_CONFIGURATION, true)) { Map<String, String> hadoopConfMap = Maps.newHashMap(); Configuration hadoopConf = new Configuration(); hadoopConf.addResource("gobblin-site.xml"); addBrokerKeys(hadoopConfMap, hadoopConf); config = config.withFallback(ConfigFactory.parseMap(hadoopConfMap)); } return config; }
private void cleanUpJobs(HelixManager helixManager) { // Clean up existing jobs TaskDriver taskDriver = new TaskDriver(helixManager); Map<String, WorkflowConfig> workflows = taskDriver.getWorkflows(); boolean cleanupDistJobs = ConfigUtils.getBoolean(this.config, GobblinClusterConfigurationKeys.CLEAN_ALL_DIST_JOBS, GobblinClusterConfigurationKeys.DEFAULT_CLEAN_ALL_DIST_JOBS); for (Map.Entry<String, WorkflowConfig> entry : workflows.entrySet()) { String workflowName = entry.getKey(); if (workflowName.contains(GobblinClusterConfigurationKeys.PLANNING_CONF_PREFIX) || workflowName.contains(GobblinClusterConfigurationKeys.ACTUAL_JOB_NAME_PREFIX)) { if (!cleanupDistJobs) { log.info("Distributed job {} won't be deleted.", workflowName); continue; } } WorkflowConfig workflowConfig = entry.getValue(); // request delete if not already requested if (workflowConfig.getTargetState() != TargetState.DELETE) { taskDriver.delete(workflowName); log.info("Requested delete of workflowName {}", workflowName); } } }
public GobblinClusterManager(String clusterName, String applicationId, Config config, Optional<Path> appWorkDirOptional) throws Exception { this.clusterName = clusterName; this.config = config; this.isStandaloneMode = ConfigUtils.getBoolean(config, GobblinClusterConfigurationKeys.STANDALONE_CLUSTER_MODE_KEY, GobblinClusterConfigurationKeys.DEFAULT_STANDALONE_CLUSTER_MODE); this.applicationId = applicationId; initializeHelixManager(); this.fs = buildFileSystem(config); this.appWorkDir = appWorkDirOptional.isPresent() ? appWorkDirOptional.get() : GobblinClusterUtils.getAppWorkDirPathFromConfig(config, this.fs, clusterName, applicationId); initializeAppLauncherAndServices(); }
/** * Load a {@link Properties} compatible path using fallback as fallback. * @return The {@link Config} in path with fallback as fallback. * @throws IOException */ private Config loadJavaPropsWithFallback(Path propertiesPath, Config fallback) throws IOException { PropertiesConfiguration propertiesConfiguration = new PropertiesConfiguration(); try (InputStreamReader inputStreamReader = new InputStreamReader(this.fs.open(propertiesPath), Charsets.UTF_8)) { propertiesConfiguration.setDelimiterParsingDisabled(ConfigUtils.getBoolean(fallback, PROPERTY_DELIMITER_PARSING_ENABLED_KEY, DEFAULT_PROPERTY_DELIMITER_PARSING_ENABLED_KEY)); propertiesConfiguration.load(inputStreamReader); Config configFromProps = ConfigUtils.propertiesToConfig(ConfigurationConverter.getProperties(propertiesConfiguration)); return ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, PathUtils.getPathWithoutSchemeAndAuthority(propertiesPath).toString())) .withFallback(configFromProps) .withFallback(fallback); } catch (ConfigurationException ce) { throw new IOException(ce); } }
public AzkabanProjectConfig(JobSpec jobSpec) { // Extract config objects this.jobSpec = jobSpec; Config defaultConfig = ConfigFactory.load(ServiceAzkabanConfigKeys.DEFAULT_AZKABAN_PROJECT_CONFIG_FILE); Config config = jobSpec.getConfig().withFallback(defaultConfig); // Azkaban Infrastructure this.azkabanServerUrl = config.getString(ServiceAzkabanConfigKeys.AZKABAN_SERVER_URL_KEY); // Azkaban Project Metadata this.azkabanProjectName = constructProjectName(jobSpec, config); this.azkabanProjectDescription = config.getString(ServiceAzkabanConfigKeys.AZKABAN_PROJECT_DESCRIPTION_KEY); this.azkabanProjectFlowName = config.getString(ServiceAzkabanConfigKeys.AZKABAN_PROJECT_FLOW_NAME_KEY); this.azkabanGroupAdminUsers = ConfigUtils.getString(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_GROUP_ADMINS_KEY, ""); this.azkabanUserToProxy = Optional.ofNullable(ConfigUtils.getString(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_USER_TO_PROXY_KEY, null)); // Azkaban Project Zip this.azkabanZipJarNames = Optional.ofNullable(ConfigUtils.getStringList(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_NAMES_KEY)); this.azkabanZipJarUrlTemplate = Optional.ofNullable(ConfigUtils.getString(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_URL_TEMPLATE_KEY, null)); this.azkabanZipJarVersion = Optional.ofNullable(ConfigUtils.getString(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_VERSION_KEY, null)); if (config.hasPath(ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY) && StringUtils.isNotBlank(config.getString(ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY))) { this.azkabanZipAdditionalFiles = Optional.ofNullable( ConfigUtils.getStringList(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY)); } else { this.azkabanZipAdditionalFiles = Optional.empty(); } this.failIfJarNotFound = ConfigUtils.getBoolean(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_FAIL_IF_JARNOTFOUND_KEY, false); }
public StageableTableMetadata(Config config, @Nullable Table referenceTable) { Preconditions.checkArgument(config.hasPath(DESTINATION_TABLE_KEY), String.format("Key %s is not specified", DESTINATION_TABLE_KEY)); Preconditions.checkArgument(config.hasPath(DESTINATION_DB_KEY), String.format("Key %s is not specified", DESTINATION_DB_KEY)); Preconditions.checkArgument(config.hasPath(DESTINATION_DATA_PATH_KEY), String.format("Key %s is not specified", DESTINATION_DATA_PATH_KEY)); // Required this.destinationTableName = referenceTable == null ? config.getString(DESTINATION_TABLE_KEY) : HiveDataset.resolveTemplate(config.getString(DESTINATION_TABLE_KEY), referenceTable); this.destinationStagingTableName = String.format("%s_%s", this.destinationTableName, "staging"); // Fixed and non-configurable this.destinationDbName = referenceTable == null ? config.getString(DESTINATION_DB_KEY) : HiveDataset.resolveTemplate(config.getString(DESTINATION_DB_KEY), referenceTable); this.destinationDataPath = referenceTable == null ? config.getString(DESTINATION_DATA_PATH_KEY) : HiveDataset.resolveTemplate(config.getString(DESTINATION_DATA_PATH_KEY), referenceTable); // Optional this.destinationTableProperties = convertKeyValueListToProperties(ConfigUtils.getStringList(config, DESTINATION_TABLE_PROPERTIES_LIST_KEY)); this.clusterBy = ConfigUtils.getStringList(config, CLUSTER_BY_KEY); this.numBuckets = Optional.fromNullable(ConfigUtils.getInt(config, NUM_BUCKETS_KEY, null)); this.hiveRuntimeProperties = convertKeyValueListToProperties(ConfigUtils.getStringList(config, HIVE_RUNTIME_PROPERTIES_LIST_KEY)); this.evolutionEnabled = ConfigUtils.getBoolean(config, EVOLUTION_ENABLED, false); this.rowLimit = Optional.fromNullable(ConfigUtils.getInt(config, ROW_LIMIT_KEY, null)); this.sourceDataPathIdentifier = ConfigUtils.getStringList(config, SOURCE_DATA_PATH_IDENTIFIER_KEY); }
@Override public void run(String[] args) throws Exception { CliObjectFactory<Command> factory = new ConstructorAndPublicMethodsCliObjectFactory<>(Command.class); Command command = factory.buildObject(args, 1, true, args[0]); FileSystem fs = FileSystem.get(new Configuration()); FSDataInputStream inputStream = fs.open(command.path); Config config = ConfigFactory.parseReader(new InputStreamReader(inputStream, Charset.defaultCharset())); Preconditions.checkNotNull(config.getObject(SOURCE_KEY)); Preconditions.checkNotNull(config.getObject(DESTINATION_KEY)); DatasetStateStore dstDatasetStateStore = DatasetStateStore.buildDatasetStateStore(config.getConfig(DESTINATION_KEY)); DatasetStateStore srcDatasetStateStore = DatasetStateStore.buildDatasetStateStore(config.getConfig(SOURCE_KEY)); Map<String, JobState.DatasetState> map; // if migrating state for all jobs then list the store names (job names) and copy the current jst files if (ConfigUtils.getBoolean(config, MIGRATE_ALL_JOBS, Boolean.valueOf(DEFAULT_MIGRATE_ALL_JOBS))) { List<String> jobNames = srcDatasetStateStore.getStoreNames(Predicates.alwaysTrue()); for (String jobName : jobNames) { migrateStateForJob(srcDatasetStateStore, dstDatasetStateStore, jobName, command.deleteSourceStateStore); } } else { Preconditions.checkNotNull(config.getString(JOB_NAME_KEY)); migrateStateForJob(srcDatasetStateStore, dstDatasetStateStore, config.getString(JOB_NAME_KEY), command.deleteSourceStateStore); } }
private static boolean isNodeForkable(DagNode<JobExecutionPlan> dagNode) { Config jobConfig = dagNode.getValue().getJobSpec().getConfig(); return ConfigUtils.getBoolean(jobConfig, ConfigurationKeys.JOB_FORK_ON_CONCAT, false); }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }
/** * A method to add tracking event configurations to a JobSpec. * This enables {@link org.apache.gobblin.metrics.GobblinTrackingEvent}s * to be emitted from each Gobblin job orchestrated by Gobblin-as-a-Service, which will then be used for tracking the * execution status of the job. * @param jobSpec representing a fully resolved {@link JobSpec}. */ private static void addTrackingEventConfig(JobSpec jobSpec, Config sysConfig) { Config reportingConfig = ConfigUtils.getConfig(sysConfig, ConfigurationKeys.METRICS_REPORTING_CONFIGURATIONS_PREFIX, ConfigFactory.empty()); if (!reportingConfig.isEmpty()) { Config jobConfig = jobSpec.getConfig().withFallback(reportingConfig.atPath(ConfigurationKeys.METRICS_REPORTING_CONFIGURATIONS_PREFIX)); boolean isSchemaRegistryEnabled = ConfigUtils.getBoolean(sysConfig, ConfigurationKeys.METRICS_REPORTING_KAFKA_USE_SCHEMA_REGISTRY, false); if (isSchemaRegistryEnabled) { String schemaRegistryUrl = ConfigUtils.getString(sysConfig, KafkaSchemaRegistry.KAFKA_SCHEMA_REGISTRY_URL, ""); if (!Strings.isNullOrEmpty(schemaRegistryUrl)) { jobConfig = jobConfig.withValue(KafkaSchemaRegistry.KAFKA_SCHEMA_REGISTRY_URL, ConfigValueFactory.fromAnyRef(schemaRegistryUrl)); } String schemaOverrideNamespace = ConfigUtils .getString(sysConfig, KafkaSchemaRegistryConfigurationKeys.KAFKA_SCHEMA_REGISTRY_OVERRIDE_NAMESPACE, ""); if (!Strings.isNullOrEmpty(schemaOverrideNamespace)) { jobConfig = jobConfig.withValue(KafkaSchemaRegistryConfigurationKeys.KAFKA_SCHEMA_REGISTRY_OVERRIDE_NAMESPACE, ConfigValueFactory.fromAnyRef(schemaOverrideNamespace)); } } jobSpec.setConfig(jobConfig); } }
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); CouchbaseEnvironment couchbaseEnvironment = CouchbaseEnvironmentFactory.getInstance(config); //TODO: Read config to decide whether to build a blocking writer or an async writer double failureAllowance = ConfigUtils.getDouble(config, CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_CONFIG, CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; boolean retriesEnabled = ConfigUtils.getBoolean(config, CouchbaseWriterConfigurationKeys.RETRIES_ENABLED, CouchbaseWriterConfigurationKeys.RETRIES_ENABLED_DEFAULT); int maxRetries = ConfigUtils.getInt(config, CouchbaseWriterConfigurationKeys.MAX_RETRIES, CouchbaseWriterConfigurationKeys.MAX_RETRIES_DEFAULT); // build an async couchbase writer AsyncDataWriter couchbaseWriter = new CouchbaseWriter(couchbaseEnvironment, config); return AsyncWriterManager.builder() .asyncDataWriter(couchbaseWriter) .failureAllowanceRatio(failureAllowance) .retriesEnabled(retriesEnabled) .numRetries(maxRetries) .config(config) .build(); } }