public SequentialBasedBatchAccumulator(Config config) { this(ConfigUtils.getLong(config, Batch.BATCH_SIZE, Batch.BATCH_SIZE_DEFAULT), ConfigUtils.getLong(config, Batch.BATCH_TTL, Batch.BATCH_TTL_DEFAULT), ConfigUtils.getLong(config, Batch.BATCH_QUEUE_CAPACITY, Batch.BATCH_QUEUE_CAPACITY_DEFAULT)); }
public GobblinHelixDistributeJobExecutionLauncher(Builder builder) throws Exception { if (builder.taskDriverHelixManager.isPresent()) { this.planningJobHelixManager = builder.taskDriverHelixManager.get(); } else { this.planningJobHelixManager = builder.jobHelixManager; } this.helixTaskDriver = new TaskDriver(this.planningJobHelixManager); this.sysProps = builder.sysProps; this.jobPlanningProps = builder.jobPlanningProps; this.jobSubmitted = false; Config combined = ConfigUtils.propertiesToConfig(jobPlanningProps) .withFallback(ConfigUtils.propertiesToConfig(sysProps)); this.workFlowExpiryTimeSeconds = ConfigUtils.getLong(combined, GobblinClusterConfigurationKeys.HELIX_WORKFLOW_EXPIRY_TIME_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_EXPIRY_TIME_SECONDS); this.planningJobLauncherMetrics = builder.planningJobLauncherMetrics; this.helixMetrics = builder.helixMetrics; this.jobsMapping = builder.jobsMapping; this.helixJobStopTimeoutSeconds = ConfigUtils.getLong(combined, GobblinClusterConfigurationKeys.HELIX_JOB_STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_JOB_STOP_TIMEOUT_SECONDS); }
public FineGrainedWatermarkTracker(Config config) { _watermarksMap = new HashMap<>(); _sweepIntervalMillis = ConfigUtils.getLong(config, WATERMARK_TRACKER_SWEEP_INTERVAL_MS, WATERMARK_TRACKER_SWEEP_INTERVAL_MS_DEFAULT); _stabilityCheckIntervalMillis = ConfigUtils.getLong(config, WATERMARK_TRACKER_STABILITY_CHECK_INTERVAL_MS, WATERMARK_TRACKER_STABILITY_CHECK_INTERVAL_MS_DEFAULT); _watermarkLagThreshold = ConfigUtils.getLong(config, WATERMARK_TRACKER_LAG_THRESHOLD, WATERMARK_TRACKER_LAG_THRESHOLD_DEFAULT); _instrumentationEnabled = GobblinMetrics.isEnabled(config); _closer = Closer.create(); _metricContext = _closer.register(Instrumented.getMetricContext(ConfigUtils.configToState(config), this.getClass())); regenerateMetrics(); _started = new AtomicBoolean(false); _abort = new AtomicBoolean(false); _sweeper = new Runnable() { @Override public void run() { sweep(); } }; _stabilityChecker = new Runnable() { @Override public void run() { checkStability(); } }; }
public ScheduledJobConfigurationManager(EventBus eventBus, Config config) { super(eventBus, config); this.jobSpecs = Maps.newHashMap(); this.refreshIntervalInSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.JOB_SPEC_REFRESH_INTERVAL, DEFAULT_JOB_SPEC_REFRESH_INTERVAL); this.fetchJobSpecExecutor = Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); this.aliasResolver = new ClassAliasResolver<>(SpecConsumer.class); try { String specConsumerClassName = GobblinClusterConfigurationKeys.DEFAULT_SPEC_CONSUMER_CLASS; if (config.hasPath(GobblinClusterConfigurationKeys.SPEC_CONSUMER_CLASS_KEY)) { specConsumerClassName = config.getString(GobblinClusterConfigurationKeys.SPEC_CONSUMER_CLASS_KEY); } LOGGER.info("Using SpecConsumer ClassNameclass name/alias " + specConsumerClassName); this._specConsumer = (SpecConsumer) ConstructorUtils .invokeConstructor(Class.forName(this.aliasResolver.resolve(specConsumerClassName)), config); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException(e); } }
public StreamingJobConfigurationManager(EventBus eventBus, Config config, MutableJobCatalog jobCatalog) { super(eventBus, config); this.stopTimeoutSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_STOP_TIMEOUT_SECONDS); this.fetchJobSpecExecutor = Executors.newSingleThreadExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); String specExecutorInstanceConsumerClassName = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.SPEC_CONSUMER_CLASS_KEY, GobblinClusterConfigurationKeys.DEFAULT_STREAMING_SPEC_CONSUMER_CLASS); LOGGER.info("Using SpecConsumer ClassNameclass name/alias " + specExecutorInstanceConsumerClassName); try { ClassAliasResolver<SpecConsumer> aliasResolver = new ClassAliasResolver<>(SpecConsumer.class); this.specConsumer = (SpecConsumer) GobblinConstructorUtils.invokeFirstConstructor( Class.forName(aliasResolver.resolve(specExecutorInstanceConsumerClassName)), ImmutableList.<Object>of(config, jobCatalog), ImmutableList.<Object>of(config)); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException("Could not construct SpecConsumer " + specExecutorInstanceConsumerClassName, e); } }
private void configureIfNeeded(Config config) { if (!configured.get()) { num_parallelism = ConfigUtils.getInt(config, "source.numParallelism", DEFAULT_NUM_PARALLELISM); namespace = ConfigUtils.getString(config, "source.namespace", DEFAULT_NAMESPACE); table = ConfigUtils.getString(config, "source.table", DEFAULT_TABLE); numRecordsPerExtract = ConfigUtils.getInt(config, "source.numRecordsPerExtract", DEFAULT_NUM_RECORDS_PER_EXTRACT); sleepTimePerRecord = ConfigUtils.getLong(config, "source.sleepTimePerRecordMillis", DEFAULT_SLEEP_TIME_PER_RECORD_MILLIS); streaming = (ConfigUtils.getString(config, "task.executionMode", "BATCH").equalsIgnoreCase("STREAMING")); if (streaming) { numRecordsPerExtract = Integer.MAX_VALUE; } configured.set(true); } }
@Override public SharedResourceFactoryResponse<ThrottlingPolicy> createResource(SharedResourcesBroker<ThrottlingServerScopes> broker, ScopedConfigView<ThrottlingServerScopes, SharedLimiterKey> configView) throws NotConfiguredException { Config config = configView.getConfig(); if (!config.hasPath(POLICY_KEY)) { if (config.hasPath(FAIL_ON_UNKNOWN_RESOURCE_ID) && config.getBoolean(FAIL_ON_UNKNOWN_RESOURCE_ID)) { throw new NotConfiguredException("Missing key " + POLICY_KEY); } else { return new TTLResourceEntry<ThrottlingPolicy>(new NoopPolicy(), ConfigUtils.getLong(config, RELOAD_FREQUENCY_KEY, DEFAULT_RELOAD_FREQUENCY), false); } } try { SpecificPolicyFactory factory = POLICY_CLASS_RESOLVER.resolveClass(config.getString(POLICY_KEY)).newInstance(); return new TTLResourceEntry<>(factory.createPolicy(configView.getKey(), broker, config), ConfigUtils.getLong(config, RELOAD_FREQUENCY_KEY, DEFAULT_RELOAD_FREQUENCY), false); } catch (ReflectiveOperationException roe) { throw new RuntimeException(roe); } }
public CouchbaseWriter(CouchbaseEnvironment couchbaseEnvironment, Config config) { List<String> hosts = ConfigUtils.getStringList(config, CouchbaseWriterConfigurationKeys.BOOTSTRAP_SERVERS); String password = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.PASSWORD, ""); String bucketName = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.BUCKET, CouchbaseWriterConfigurationKeys.BUCKET_DEFAULT); _cluster = CouchbaseCluster.create(couchbaseEnvironment, hosts); if(!password.isEmpty()) { _bucket = _cluster.openBucket(bucketName, password, Collections.<Transcoder<? extends Document, ?>>singletonList(_tupleDocumentTranscoder)); } else { _bucket = _cluster.openBucket(bucketName, Collections.<Transcoder<? extends Document, ?>>singletonList(_tupleDocumentTranscoder)); } _operationTimeout = ConfigUtils.getLong(config, CouchbaseWriterConfigurationKeys.OPERATION_TIMEOUT_MILLIS, CouchbaseWriterConfigurationKeys.OPERATION_TIMEOUT_DEFAULT); _operationTimeunit = TimeUnit.MILLISECONDS; _defaultWriteResponseMapper = new GenericWriteResponseWrapper<>(); log.info("Couchbase writer configured with: hosts: {}, bucketName: {}, operationTimeoutInMillis: {}", hosts, bucketName, _operationTimeout); }
this.workFlowExpiryTimeSeconds = ConfigUtils.getLong(jobConfig, GobblinClusterConfigurationKeys.HELIX_WORKFLOW_EXPIRY_TIME_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_EXPIRY_TIME_SECONDS); this.helixJobStopTimeoutSeconds = ConfigUtils.getLong(jobConfig, GobblinClusterConfigurationKeys.HELIX_JOB_STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_JOB_STOP_TIMEOUT_SECONDS);
long commitIntervalMillis = ConfigUtils.getLong(config, TaskConfigurationKeys.STREAMING_WATERMARK_COMMIT_INTERVAL_MILLIS, TaskConfigurationKeys.DEFAULT_STREAMING_WATERMARK_COMMIT_INTERVAL_MILLIS);
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); long commitTimeoutMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_CONFIG, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_DEFAULT); long commitStepWaitTimeMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_CONFIG, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_DEFAULT); double failureAllowance = ConfigUtils.getDouble(config, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_CONFIG, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; return AsyncWriterManager.builder() .config(config) .commitTimeoutMillis(commitTimeoutMillis) .commitStepWaitTimeInMillis(commitStepWaitTimeMillis) .failureAllowanceRatio(failureAllowance) .retriesEnabled(false) .asyncDataWriter(getAsyncDataWriter(taskProps)).maxOutstandingWrites(10000) .build(); } }
/** * creates a new {@link BasicDataSource} * @param config the properties used for datasource instantiation * @return */ public static BasicDataSource newDataSource(Config config) { BasicDataSource basicDataSource = new BasicDataSource(); PasswordManager passwordManager = PasswordManager.getInstance(ConfigUtils.configToProperties(config)); basicDataSource.setDriverClassName(ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_JDBC_DRIVER_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_JDBC_DRIVER)); // MySQL server can timeout a connection so need to validate connections before use basicDataSource.setValidationQuery("select 1"); basicDataSource.setTestOnBorrow(true); basicDataSource.setDefaultAutoCommit(false); basicDataSource.setTimeBetweenEvictionRunsMillis(60000); basicDataSource.setUrl(config.getString(ConfigurationKeys.STATE_STORE_DB_URL_KEY)); basicDataSource.setUsername(passwordManager.readPassword( config.getString(ConfigurationKeys.STATE_STORE_DB_USER_KEY))); basicDataSource.setPassword(passwordManager.readPassword( config.getString(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY))); basicDataSource.setMinEvictableIdleTimeMillis( ConfigUtils.getLong(config, ConfigurationKeys.STATE_STORE_DB_CONN_MIN_EVICTABLE_IDLE_TIME_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_CONN_MIN_EVICTABLE_IDLE_TIME)); return basicDataSource; }
/** * Build a {@link DataWriter}. * * @throws IOException if there is anything wrong building the writer * @return the built {@link DataWriter} */ @Override public DataWriter<D> build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); long commitTimeoutMillis = ConfigUtils.getLong(config, KafkaWriterConfigurationKeys.COMMIT_TIMEOUT_MILLIS_CONFIG, KafkaWriterConfigurationKeys.COMMIT_TIMEOUT_MILLIS_DEFAULT); long commitStepWaitTimeMillis = ConfigUtils.getLong(config, KafkaWriterConfigurationKeys.COMMIT_STEP_WAIT_TIME_CONFIG, KafkaWriterConfigurationKeys.COMMIT_STEP_WAIT_TIME_DEFAULT); double failureAllowance = ConfigUtils.getDouble(config, KafkaWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_CONFIG, KafkaWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; return AsyncWriterManager.builder() .config(config) .commitTimeoutMillis(commitTimeoutMillis) .commitStepWaitTimeInMillis(commitStepWaitTimeMillis) .failureAllowanceRatio(failureAllowance) .retriesEnabled(false) .asyncDataWriter(getAsyncDataWriter(taskProps)) .build(); } }
public SequentialBasedBatchAccumulator(Config config) { this(ConfigUtils.getLong(config, Batch.BATCH_SIZE, Batch.BATCH_SIZE_DEFAULT), ConfigUtils.getLong(config, Batch.BATCH_TTL, Batch.BATCH_TTL_DEFAULT), ConfigUtils.getLong(config, Batch.BATCH_QUEUE_CAPACITY, Batch.BATCH_QUEUE_CAPACITY_DEFAULT)); }
public ScheduledJobConfigurationManager(EventBus eventBus, Config config) { super(eventBus, config); this.jobSpecs = Maps.newHashMap(); this.refreshIntervalInSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.JOB_SPEC_REFRESH_INTERVAL, DEFAULT_JOB_SPEC_REFRESH_INTERVAL); this.fetchJobSpecExecutor = Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); this.aliasResolver = new ClassAliasResolver<>(SpecConsumer.class); try { String specConsumerClassName = GobblinClusterConfigurationKeys.DEFAULT_SPEC_CONSUMER_CLASS; if (config.hasPath(GobblinClusterConfigurationKeys.SPEC_CONSUMER_CLASS_KEY)) { specConsumerClassName = config.getString(GobblinClusterConfigurationKeys.SPEC_CONSUMER_CLASS_KEY); } LOGGER.info("Using SpecConsumer ClassNameclass name/alias " + specConsumerClassName); this._specConsumer = (SpecConsumer) ConstructorUtils .invokeConstructor(Class.forName(this.aliasResolver.resolve(specConsumerClassName)), config); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException(e); } }
public GobblinHelixDistributeJobExecutionLauncher(Builder builder) throws Exception { this.helixManager = builder.manager; this.helixTaskDriver = new TaskDriver(this.helixManager); this.sysProps = builder.sysProps; this.jobPlanningProps = builder.jobPlanningProps; this.jobSubmitted = false; Config combined = ConfigUtils.propertiesToConfig(jobPlanningProps) .withFallback(ConfigUtils.propertiesToConfig(sysProps)); Config stateStoreJobConfig = combined .withValue(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigValueFactory.fromAnyRef( new URI(builder.appWorkDir.toUri().getScheme(), null, builder.appWorkDir.toUri().getHost(), builder.appWorkDir.toUri().getPort(), null, null, null).toString())); this.stateStores = new StateStores(stateStoreJobConfig, builder.appWorkDir, PLANNING_TASK_STATE_DIR_NAME, builder.appWorkDir, PLANNING_WORK_UNIT_DIR_NAME, builder.appWorkDir, PLANNING_JOB_STATE_DIR_NAME); this.workFlowExpiryTimeSeconds = ConfigUtils.getLong(combined, GobblinClusterConfigurationKeys.HELIX_WORKFLOW_EXPIRY_TIME_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_EXPIRY_TIME_SECONDS); }
public StreamingJobConfigurationManager(EventBus eventBus, Config config, MutableJobCatalog jobCatalog) { super(eventBus, config); this.stopTimeoutSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_STOP_TIMEOUT_SECONDS); this.fetchJobSpecExecutor = Executors.newSingleThreadExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); String specExecutorInstanceConsumerClassName = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.SPEC_CONSUMER_CLASS_KEY, GobblinClusterConfigurationKeys.DEFAULT_STREAMING_SPEC_CONSUMER_CLASS); LOGGER.info("Using SpecConsumer ClassNameclass name/alias " + specExecutorInstanceConsumerClassName); try { ClassAliasResolver<SpecConsumer> aliasResolver = new ClassAliasResolver<>(SpecConsumer.class); this.specConsumer = (SpecConsumer) GobblinConstructorUtils.invokeFirstConstructor( Class.forName(aliasResolver.resolve(specExecutorInstanceConsumerClassName)), ImmutableList.<Object>of(config, jobCatalog), ImmutableList.<Object>of(config)); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException("Could not construct SpecConsumer " + specExecutorInstanceConsumerClassName, e); } }
private void configureIfNeeded(Config config) { if (!configured.get()) { num_parallelism = ConfigUtils.getInt(config, "source.numParallelism", DEFAULT_NUM_PARALLELISM); namespace = ConfigUtils.getString(config, "source.namespace", DEFAULT_NAMESPACE); table = ConfigUtils.getString(config, "source.table", DEFAULT_TABLE); numRecordsPerExtract = ConfigUtils.getInt(config, "source.numRecordsPerExtract", DEFAULT_NUM_RECORDS_PER_EXTRACT); sleepTimePerRecord = ConfigUtils.getLong(config, "source.sleepTimePerRecordMillis", DEFAULT_SLEEP_TIME_PER_RECORD_MILLIS); streaming = (ConfigUtils.getString(config, "task.executionMode", "BATCH").equalsIgnoreCase("STREAMING")); if (streaming) { numRecordsPerExtract = Integer.MAX_VALUE; } configured.set(true); } }
@Override public SharedResourceFactoryResponse<ThrottlingPolicy> createResource(SharedResourcesBroker<ThrottlingServerScopes> broker, ScopedConfigView<ThrottlingServerScopes, SharedLimiterKey> configView) throws NotConfiguredException { Config config = configView.getConfig(); if (!config.hasPath(POLICY_KEY)) { if (config.hasPath(FAIL_ON_UNKNOWN_RESOURCE_ID) && config.getBoolean(FAIL_ON_UNKNOWN_RESOURCE_ID)) { throw new NotConfiguredException("Missing key " + POLICY_KEY); } else { return new TTLResourceEntry<ThrottlingPolicy>(new NoopPolicy(), ConfigUtils.getLong(config, RELOAD_FREQUENCY_KEY, DEFAULT_RELOAD_FREQUENCY), false); } } try { SpecificPolicyFactory factory = POLICY_CLASS_RESOLVER.resolveClass(config.getString(POLICY_KEY)).newInstance(); return new TTLResourceEntry<>(factory.createPolicy(configView.getKey(), broker, config), ConfigUtils.getLong(config, RELOAD_FREQUENCY_KEY, DEFAULT_RELOAD_FREQUENCY), false); } catch (ReflectiveOperationException roe) { throw new RuntimeException(roe); } }
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); long commitTimeoutMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_CONFIG, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_DEFAULT); long commitStepWaitTimeMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_CONFIG, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_DEFAULT); double failureAllowance = ConfigUtils.getDouble(config, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_CONFIG, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; return AsyncWriterManager.builder() .config(config) .commitTimeoutMillis(commitTimeoutMillis) .commitStepWaitTimeInMillis(commitStepWaitTimeMillis) .failureAllowanceRatio(failureAllowance) .retriesEnabled(false) .asyncDataWriter(getAsyncDataWriter(taskProps)).maxOutstandingWrites(10000) .build(); } }