@Override public String toString() { return Objects.toStringHelper(JobContext.class.getSimpleName()).add("jobName", getJobName()) .add("jobId", getJobId()).add("jobState", getJobState()).toString(); } }
public void launchJob(@Nullable JobListener jobListener) throws JobException { this.jobListener = jobListener; boolean isLaunched = false; this.runningMap.putIfAbsent(this.jobContext.getJobName(), false); try { if (this.runningMap.replace(this.jobContext.getJobName(), false, true)) { LOGGER.info ("Job {} will be executed, add into running map.", this.jobContext.getJobId()); isLaunched = true; super.launchJob(jobListener); } else { LOGGER.warn ("Job {} will not be executed because other jobs are still running.", this.jobContext.getJobId()); } } finally { if (isLaunched) { if (this.runningMap.replace(this.jobContext.getJobName(), true, false)) { LOGGER.info ("Job {} is done, remove from running map.", this.jobContext.getJobId()); } else { throw new IllegalStateException("A launched job should have running state equal to true in the running map."); } } } }
/** * Add a single {@link WorkUnit} (flattened). */ private void addWorkUnit(WorkUnit workUnit, ParallelRunner stateSerDeRunner, Map<String, TaskConfig> taskConfigMap) throws IOException { String workUnitFilePath = persistWorkUnit( new Path(this.inputWorkUnitDir, this.jobContext.getJobId()), workUnit, stateSerDeRunner); Map<String, String> rawConfigMap = Maps.newHashMap(); rawConfigMap.put(GobblinClusterConfigurationKeys.WORK_UNIT_FILE_PATH, workUnitFilePath); rawConfigMap.put(ConfigurationKeys.JOB_NAME_KEY, this.jobContext.getJobName()); rawConfigMap.put(ConfigurationKeys.JOB_ID_KEY, this.jobContext.getJobId()); rawConfigMap.put(ConfigurationKeys.TASK_ID_KEY, workUnit.getId()); rawConfigMap.put(GobblinClusterConfigurationKeys.TASK_SUCCESS_OPTIONAL_KEY, "true"); taskConfigMap.put(workUnit.getId(), TaskConfig.Builder.from(rawConfigMap)); }
new Path(this.jobProps.getProperty(ConfigurationKeys.MR_JOB_ROOT_DIR_KEY), this.jobContext.getJobName()), this.jobContext.getJobId()); if (this.fs.exists(this.mrJobDir)) { LOG.warn("Job working directory already exists for job " + this.jobContext.getJobName()); this.fs.delete(this.mrJobDir, true); this.job = Job.getInstance(this.conf, JOB_NAME_PREFIX + this.jobContext.getJobName());
protected void setTaskStagingAndOutputDirs() { // Add jobId to writer staging dir if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { String writerStagingDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, writerStagingDirWithJobId); } // Add jobId to writer output dir if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { String writerOutputDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), this.getJobName()), this.jobId) .toString(); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, writerOutputDirWithJobId); } // Add jobId to task data root dir if (this.jobState.contains(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)) { String taskDataRootDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY, taskDataRootDirWithJobId); setTaskStagingDir(); setTaskOutputDir(); } else { LOG.warn("Property " + ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY + " is missing."); } }
@Override protected void runWorkUnits(List<WorkUnit> workUnits) throws Exception { String jobName = this.jobContext.getJobName(); JobState jobState = this.jobContext.getJobState();
MDC.put(ConfigurationKeys.JOB_NAME_KEY, this.jobContext.getJobName()); MDC.put(ConfigurationKeys.JOB_KEY_KEY, this.jobContext.getJobKey()); TimingEvent launchJobTimer = this.eventSubmitter.getTimingEvent(TimingEvent.LauncherTimings.FULL_JOB_EXECUTION);
@Override public String toString() { return Objects.toStringHelper(JobContext.class.getSimpleName()).add("jobName", getJobName()) .add("jobId", getJobId()).add("jobState", getJobState()).toString(); } }
public void launchJob(@Nullable JobListener jobListener) throws JobException { this.jobListener = jobListener; boolean isLaunched = false; this.runningMap.putIfAbsent(this.jobContext.getJobName(), false); try { if (this.runningMap.replace(this.jobContext.getJobName(), false, true)) { LOGGER.info ("Job {} will be executed, add into running map.", this.jobContext.getJobId()); isLaunched = true; super.launchJob(jobListener); } else { LOGGER.warn ("Job {} will not be executed because other jobs are still running.", this.jobContext.getJobId()); } } finally { if (isLaunched) { if (this.runningMap.replace(this.jobContext.getJobName(), true, false)) { LOGGER.info ("Job {} is done, remove from running map.", this.jobContext.getJobId()); } else { throw new IllegalStateException("A launched job should have running state equal to true in the running map."); } } } }
/** * Add a single {@link WorkUnit} (flattened). */ private void addWorkUnit(WorkUnit workUnit, ParallelRunner stateSerDeRunner, Map<String, TaskConfig> taskConfigMap) throws IOException { String workUnitFilePath = persistWorkUnit( new Path(this.inputWorkUnitDir, this.jobContext.getJobId()), workUnit, stateSerDeRunner); Map<String, String> rawConfigMap = Maps.newHashMap(); rawConfigMap.put(GobblinClusterConfigurationKeys.WORK_UNIT_FILE_PATH, workUnitFilePath); rawConfigMap.put(ConfigurationKeys.JOB_NAME_KEY, this.jobContext.getJobName()); rawConfigMap.put(ConfigurationKeys.JOB_ID_KEY, this.jobContext.getJobId()); rawConfigMap.put(ConfigurationKeys.TASK_ID_KEY, workUnit.getId()); rawConfigMap.put(GobblinClusterConfigurationKeys.TASK_SUCCESS_OPTIONAL_KEY, "true"); taskConfigMap.put(workUnit.getId(), TaskConfig.Builder.from(rawConfigMap)); }
new Path(this.jobProps.getProperty(ConfigurationKeys.MR_JOB_ROOT_DIR_KEY), this.jobContext.getJobName()), this.jobContext.getJobId()); if (this.fs.exists(this.mrJobDir)) { LOG.warn("Job working directory already exists for job " + this.jobContext.getJobName()); this.fs.delete(this.mrJobDir, true); this.job = Job.getInstance(this.conf, JOB_NAME_PREFIX + this.jobContext.getJobName());
protected void setTaskStagingAndOutputDirs() { // Add jobId to writer staging dir if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { String writerStagingDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, writerStagingDirWithJobId); } // Add jobId to writer output dir if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { String writerOutputDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), this.getJobName()), this.jobId) .toString(); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, writerOutputDirWithJobId); } // Add jobId to task data root dir if (this.jobState.contains(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)) { String taskDataRootDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY, taskDataRootDirWithJobId); setTaskStagingDir(); setTaskOutputDir(); } else { LOG.warn("Property " + ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY + " is missing."); } }
@Override protected void runWorkUnits(List<WorkUnit> workUnits) throws Exception { String jobName = this.jobContext.getJobName(); JobState jobState = this.jobContext.getJobState();
MDC.put(ConfigurationKeys.JOB_NAME_KEY, this.jobContext.getJobName()); MDC.put(ConfigurationKeys.JOB_KEY_KEY, this.jobContext.getJobKey()); TimingEvent launchJobTimer = this.eventSubmitter.getTimingEvent(TimingEvent.LauncherTimings.FULL_JOB_EXECUTION);