public void setDatasetUrn(String datasetUrn) { super.setProp(ConfigurationKeys.DATASET_URN_KEY, datasetUrn); }
public void setNoJobFailure() { super.setProp(ConfigurationKeys.JOB_FAILURES_KEY, 0); }
public void incrementJobFailures() { super.setProp(ConfigurationKeys.JOB_FAILURES_KEY, Integer.parseInt(super.getProp(ConfigurationKeys.JOB_FAILURES_KEY, "0")) + 1); }
/** * Sets the {@link LauncherTypeEnum} for this {@link JobState}. */ public void setJobLauncherType(LauncherTypeEnum jobLauncherType) { this.setProp(ConfigurationKeys.JOB_LAUNCHER_TYPE_KEY, jobLauncherType.name()); }
@Override public void onJobPrepare(JobContext jobContext) throws Exception { super.onJobPrepare(jobContext); jobContext.getJobState().setProp(JOB_START_TIME, Long.toString(System.nanoTime())); jobLauncherMetrics.totalJobsLaunched.incrementAndGet(); }
/** * If {@link ConfigurationKeys#WRITER_STAGING_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_STAGING_DIR_NAME}. */ private void setTaskStagingDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_STAGING_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState .setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(workingDir, TASK_STAGING_DIR_NAME).toString()); LOG.info(String.format("Writer Staging Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR))); } }
/** * If {@link ConfigurationKeys#WRITER_OUTPUT_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_OUTPUT_DIR_NAME}. */ private void setTaskOutputDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_OUTPUT_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(workingDir, TASK_OUTPUT_DIR_NAME).toString()); LOG.info(String .format("Writer Output Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR))); } }
protected void setTaskStagingAndOutputDirs() { // Add jobId to writer staging dir if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { String writerStagingDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, writerStagingDirWithJobId); } // Add jobId to writer output dir if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { String writerOutputDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), this.getJobName()), this.jobId) .toString(); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, writerOutputDirWithJobId); } // Add jobId to task data root dir if (this.jobState.contains(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)) { String taskDataRootDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY, taskDataRootDirWithJobId); setTaskStagingDir(); setTaskOutputDir(); } else { LOG.warn("Property " + ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY + " is missing."); } }
/** * Finalize the {@link JobState} before committing the job. */ void finalizeJobStateBeforeCommit() { this.jobState.setEndTime(System.currentTimeMillis()); this.jobState.setDuration(this.jobState.getEndTime() - this.jobState.getStartTime()); for (TaskState taskState : this.jobState.getTaskStates()) { // Set fork.branches explicitly here so the rest job flow can pick it up this.jobState .setProp(ConfigurationKeys.FORK_BRANCHES_KEY, taskState.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1)); } }
this.jobState.setProp(entry.getKey(), entry.getValue().unwrapped().toString()); configuration.set(entry.getKey(), entry.getValue().unwrapped().toString());
jobState.setProp(ConfigurationKeys.JOB_TRACKING_URL_KEY, this.job.getTrackingURL());
this.jobState.setProp(Instrumented.METRIC_CONTEXT_NAME_KEY, this.jobMetricsOptional.get().getName()); } else { this.jobMetricsOptional = Optional.absent();
this.jobState.setState(JobState.RunningState.COMMITTED); this.jobState.setTaskCount(3); this.jobState.setProp("foo", "bar"); for (int i = 0; i < 3; i++) { WorkUnit workUnit = WorkUnit.createEmpty();
JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setProp("lower", "case"); dbJobStateStore.put(TEST_JOB_NAME_LOWER, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } this.fsJobStateStore.put(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
JobState jobState = new JobState(TEST_JOB_NAME1, getJobId(TEST_JOB_ID, 1)); jobState.setId(getJobId(TEST_JOB_ID, 1)); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime);
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } zkJobStateStore.put(TEST_JOB_NAME, ZkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + ZkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); // second job name for testing getting store names in a later test case jobState.setJobName(TEST_JOB_NAME2); zkJobStateStore.put(TEST_JOB_NAME2, ZkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + ZkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
/** * Sets the {@link LauncherTypeEnum} for this {@link JobState}. */ public void setJobLauncherType(LauncherTypeEnum jobLauncherType) { this.setProp(ConfigurationKeys.JOB_LAUNCHER_TYPE_KEY, jobLauncherType.name()); }
@Override public void onJobPrepare(JobContext jobContext) throws Exception { super.onJobPrepare(jobContext); jobContext.getJobState().setProp(JOB_START_TIME, Long.toString(System.nanoTime())); jobLauncherMetrics.totalJobsLaunched.incrementAndGet(); }
/** * Finalize the {@link JobState} before committing the job. */ void finalizeJobStateBeforeCommit() { this.jobState.setEndTime(System.currentTimeMillis()); this.jobState.setDuration(this.jobState.getEndTime() - this.jobState.getStartTime()); for (TaskState taskState : this.jobState.getTaskStates()) { // Set fork.branches explicitly here so the rest job flow can pick it up this.jobState .setProp(ConfigurationKeys.FORK_BRANCHES_KEY, taskState.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1)); } }