/** * Return whether staging data should be cleaned up on a per-task basis. * * @return {@code true} if staging data should be cleaned up on a per-task basis or {@code false} otherwise */ boolean shouldCleanupStagingDataPerTask() { return this.jobState.getPropAsBoolean(ConfigurationKeys.CLEANUP_STAGING_DATA_PER_TASK, ConfigurationKeys.DEFAULT_CLEANUP_STAGING_DATA_PER_TASK); }
public TaskStateCollectorService(Properties jobProps, JobState jobState, EventBus eventBus, StateStore<TaskState> taskStateStore, Path outputTaskStateDir) { this.jobState = jobState; this.eventBus = eventBus; this.taskStateStore = taskStateStore; this.outputTaskStateDir = outputTaskStateDir; this.stateSerDeRunnerThreads = Integer.parseInt(jobProps.getProperty(ParallelRunner.PARALLEL_RUNNER_THREADS_KEY, Integer.toString(ParallelRunner.DEFAULT_PARALLEL_RUNNER_THREADS))); this.outputTaskStatesCollectorIntervalSeconds = Integer.parseInt(jobProps.getProperty(ConfigurationKeys.TASK_STATE_COLLECTOR_INTERVAL_SECONDS, Integer.toString(ConfigurationKeys.DEFAULT_TASK_STATE_COLLECTOR_INTERVAL_SECONDS))); if (!StringUtils.isBlank(jobProps.getProperty(ConfigurationKeys.TASK_STATE_COLLECTOR_HANDLER_CLASS))) { String handlerTypeName = jobProps.getProperty(ConfigurationKeys.TASK_STATE_COLLECTOR_HANDLER_CLASS); try { ClassAliasResolver<TaskStateCollectorServiceHandler.TaskStateCollectorServiceHandlerFactory> aliasResolver = new ClassAliasResolver<>(TaskStateCollectorServiceHandler.TaskStateCollectorServiceHandlerFactory.class); TaskStateCollectorServiceHandler.TaskStateCollectorServiceHandlerFactory handlerFactory = aliasResolver.resolveClass(handlerTypeName).newInstance(); optionalTaskCollectorHandler = Optional.of(handlerCloser.register(handlerFactory.createHandler(this.jobState))); } catch (ReflectiveOperationException rfe) { throw new RuntimeException("Could not construct TaskCollectorHandler " + handlerTypeName, rfe); } } else { optionalTaskCollectorHandler = Optional.absent(); } isJobProceedOnCollectorServiceFailure = jobState.getPropAsBoolean(ConfigurationKeys.JOB_PROCEED_ON_TASK_STATE_COLLECOTR_SERVICE_FAILURE, defaultPolicyOnCollectorServiceFailure); }
if (jobState.getPropAsBoolean(ConfigurationKeys.WORK_UNIT_ENABLE_TRACKING_LOGS)) { Logger log = LoggerFactory.getLogger(GobblinMultiTaskAttempt.class.getName()); log.info("Work unit tracking log: {}", workUnits);
if (jobState.getPropAsBoolean(ConfigurationKeys.CLEANUP_STAGING_DATA_BY_INITIALIZER, false)) { if (jobState.getPropAsBoolean(ConfigurationKeys.CLEANUP_OLD_JOBS_DATA, ConfigurationKeys.DEFAULT_CLEANUP_OLD_JOBS_DATA)) { JobLauncherUtils.cleanUpOldJobData(jobState, LOG, jobContext.getStagingDirProvided(), jobContext.getOutputDirProvided());
if (jobState.getPropAsBoolean(ConfigurationKeys.CLEANUP_STAGING_DATA_BY_INITIALIZER, false)) {
this.parallelizeCommit = this.jobState.getPropAsBoolean(ConfigurationKeys.PARALLELIZE_DATASET_COMMIT, ConfigurationKeys.DEFAULT_PARALLELIZE_DATASET_COMMIT); this.parallelCommits = this.parallelizeCommit ? this.jobState
if (jobState.getPropAsBoolean(ConfigurationKeys.WORK_UNIT_ENABLE_TRACKING_LOGS)) { workUnitStream = workUnitStream.transform(new Function<WorkUnit, WorkUnit>() { @Nullable
/** * Return whether staging data should be cleaned up on a per-task basis. * * @return {@code true} if staging data should be cleaned up on a per-task basis or {@code false} otherwise */ boolean shouldCleanupStagingDataPerTask() { return this.jobState.getPropAsBoolean(ConfigurationKeys.CLEANUP_STAGING_DATA_PER_TASK, ConfigurationKeys.DEFAULT_CLEANUP_STAGING_DATA_PER_TASK); }
public TaskStateCollectorService(Properties jobProps, JobState jobState, EventBus eventBus, StateStore<TaskState> taskStateStore, Path outputTaskStateDir) { this.jobState = jobState; this.eventBus = eventBus; this.taskStateStore = taskStateStore; this.outputTaskStateDir = outputTaskStateDir; this.stateSerDeRunnerThreads = Integer.parseInt(jobProps.getProperty(ParallelRunner.PARALLEL_RUNNER_THREADS_KEY, Integer.toString(ParallelRunner.DEFAULT_PARALLEL_RUNNER_THREADS))); this.outputTaskStatesCollectorIntervalSeconds = Integer.parseInt(jobProps.getProperty(ConfigurationKeys.TASK_STATE_COLLECTOR_INTERVAL_SECONDS, Integer.toString(ConfigurationKeys.DEFAULT_TASK_STATE_COLLECTOR_INTERVAL_SECONDS))); if (!StringUtils.isBlank(jobProps.getProperty(ConfigurationKeys.TASK_STATE_COLLECTOR_HANDLER_CLASS))) { String handlerTypeName = jobProps.getProperty(ConfigurationKeys.TASK_STATE_COLLECTOR_HANDLER_CLASS); try { ClassAliasResolver<TaskStateCollectorServiceHandler.TaskStateCollectorServiceHandlerFactory> aliasResolver = new ClassAliasResolver<>(TaskStateCollectorServiceHandler.TaskStateCollectorServiceHandlerFactory.class); TaskStateCollectorServiceHandler.TaskStateCollectorServiceHandlerFactory handlerFactory = aliasResolver.resolveClass(handlerTypeName).newInstance(); optionalTaskCollectorHandler = Optional.of(handlerCloser.register(handlerFactory.createHandler(this.jobState))); } catch (ReflectiveOperationException rfe) { throw new RuntimeException("Could not construct TaskCollectorHandler " + handlerTypeName, rfe); } } else { optionalTaskCollectorHandler = Optional.absent(); } isJobProceedOnCollectorServiceFailure = jobState.getPropAsBoolean(ConfigurationKeys.JOB_PROCEED_ON_TASK_STATE_COLLECOTR_SERVICE_FAILURE, defaultPolicyOnCollectorServiceFailure); }
if (jobState.getPropAsBoolean(ConfigurationKeys.WORK_UNIT_ENABLE_TRACKING_LOGS)) { Logger log = LoggerFactory.getLogger(GobblinMultiTaskAttempt.class.getName()); log.info("Work unit tracking log: {}", workUnits);
if (jobState.getPropAsBoolean(ConfigurationKeys.CLEANUP_STAGING_DATA_BY_INITIALIZER, false)) { if (jobState.getPropAsBoolean(ConfigurationKeys.CLEANUP_OLD_JOBS_DATA, ConfigurationKeys.DEFAULT_CLEANUP_OLD_JOBS_DATA)) { JobLauncherUtils.cleanUpOldJobData(jobState, LOG, jobContext.getStagingDirProvided(), jobContext.getOutputDirProvided());
if (jobState.getPropAsBoolean(ConfigurationKeys.CLEANUP_STAGING_DATA_BY_INITIALIZER, false)) {
this.parallelizeCommit = this.jobState.getPropAsBoolean(ConfigurationKeys.PARALLELIZE_DATASET_COMMIT, ConfigurationKeys.DEFAULT_PARALLELIZE_DATASET_COMMIT); this.parallelCommits = this.parallelizeCommit ? this.jobState
if (jobState.getPropAsBoolean(ConfigurationKeys.WORK_UNIT_ENABLE_TRACKING_LOGS)) { workUnitStream = workUnitStream.transform(new Function<WorkUnit, WorkUnit>() { @Nullable