/** * Subclasses can override this method to do whatever processing on the {@link JobState} and its * associated {@link TaskState}s, e.g., aggregate task-level metrics into job-level metrics. */ protected void postProcessJobState(JobState jobState) { postProcessTaskStates(jobState.getTaskStates()); }
private static void cleanupStagingDataPerTask(JobState jobState) { Closer closer = Closer.create(); Map<String, ParallelRunner> parallelRunners = Maps.newHashMap(); try { for (TaskState taskState : jobState.getTaskStates()) { try { JobLauncherUtils.cleanTaskStagingData(taskState, LOG, closer, parallelRunners); } catch (IOException e) { LOG.error(String.format("Failed to clean staging data for task %s: %s", taskState.getTaskId(), e), e); } } } finally { try { closer.close(); } catch (IOException e) { LOG.error("Failed to clean staging data", e); } } }
public Map<String, String> getMetadata(JobContext jobContext, EventName eventName) { List<TaskState> taskStates = jobContext.getJobState().getTaskStates(); switch (eventName) { case JOB_COMPLETE: return ImmutableMap.of(PROCESSED_COUNT_KEY, Long.toString(EventMetadataUtils.getProcessedCount(taskStates))); case JOB_FAILED: return ImmutableMap.of(MESSAGE_KEY, EventMetadataUtils.getTaskFailureExceptions(taskStates)); default: break; } return ImmutableMap.of(); } }
/** * Submits an event for each {@link TaskState} in the given {@link JobState}. */ private void submitTaskStateEvents(JobState jobState) { // Build Job Metadata applicable for TaskStates ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); Map<String, String> jobMetadata = jobMetadataBuilder.build(); // Submit event for each TaskState for (TaskState taskState : jobState.getTaskStates()) { submitTaskStateEvent(taskState, jobMetadata); } }
/** * Remove the {@link JobMetrics} instance for the job with the given {@link JobState} instance. * * <p> * Removing a {@link JobMetrics} instance for a job will also remove the {@link TaskMetrics}s * of every tasks of the job. * </p> * @param jobState the given {@link JobState} instance */ public static void remove(JobState jobState) { remove(name(jobState)); for (TaskState taskState : jobState.getTaskStates()) { TaskMetrics.remove(taskState); } }
@Test(dependsOnMethods = "testPutIntoTaskStateStore") public void testCollectOutputTaskStates() throws Exception { this.taskStateCollectorService.runOneIteration(); Assert.assertEquals(this.jobState.getTaskStates().size(), 2); Assert.assertEquals(this.taskStateMap.size(), 2); Assert.assertEquals(this.taskStateMap.get(TASK_ID_0).getJobId(), JOB_ID); Assert.assertEquals(this.taskStateMap.get(TASK_ID_0).getTaskId(), TASK_ID_0); Assert.assertEquals(this.taskStateMap.get(TASK_ID_1).getJobId(), JOB_ID); Assert.assertEquals(this.taskStateMap.get(TASK_ID_1).getTaskId(), TASK_ID_1); }
/** * Finalize the {@link JobState} before committing the job. */ void finalizeJobStateBeforeCommit() { this.jobState.setEndTime(System.currentTimeMillis()); this.jobState.setDuration(this.jobState.getEndTime() - this.jobState.getStartTime()); for (TaskState taskState : this.jobState.getTaskStates()) { // Set fork.branches explicitly here so the rest job flow can pick it up this.jobState .setProp(ConfigurationKeys.FORK_BRANCHES_KEY, taskState.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1)); } }
@Test public void testSubmitJobExecutionEvents() { JobState mockJobState = mock(JobState.class, Mockito.RETURNS_SMART_NULLS); when(mockJobState.getState()).thenReturn(JobState.RunningState.SUCCESSFUL); when(mockJobState.getLauncherType()).thenReturn(LauncherTypeEnum.$UNKNOWN); when(mockJobState.getTrackingURL()).thenReturn(Optional.<String> absent()); TaskState mockTaskState1 = createMockTaskState(); TaskState mockTaskState2 = createMockTaskState(); when(mockJobState.getTaskStates()).thenReturn(Lists.newArrayList(mockTaskState1, mockTaskState2)); this.jobExecutionEventSubmitter.submitJobExecutionEvents(mockJobState); verify(this.mockEventSubmitter, atLeastOnce()).submit(any(String.class), any(Map.class)); }
private void verifyJobState(int run) throws IOException { List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(JOB_NAME, "current.jst"); Assert.assertEquals(datasetStateList.size(), 1); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getTaskStates().size(), DummySource.NUM_WORK_UNITS); for (TaskState taskState : jobState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getProp(FOO), BAR); // Check if the low watermark is properly kept track of int expectedLowWatermark = (run - 1) * DummySource.NUM_WORK_UNITS * DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR + taskState.getPropAsInt(WORK_UNIT_INDEX_KEY) * DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR + 1; Assert.assertEquals(taskState.getPropAsInt(ConfigurationKeys.WORK_UNIT_LOW_WATER_MARK_KEY), expectedLowWatermark); // Check if the high watermark is properly kept track of int expectedHighWatermark = expectedLowWatermark + DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR - 1; Assert.assertEquals(taskState.getPropAsInt(ConfigurationKeys.WORK_UNIT_HIGH_WATER_MARK_KEY), expectedHighWatermark); Assert.assertEquals(taskState.getPropAsInt(LAST_READ_RECORD_KEY), expectedHighWatermark); } }
public void runTestWithCommitSuccessfulTasksPolicy(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString()); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful"); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, TestSourceWithFaultyExtractor.class.getName()); jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0"); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getCompletedTasks(), 4); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith("0")) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.FAILED); } else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = this.fsDatasetStateStore.get(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith(skippedTaskSuffix)) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.PENDING);
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertNotEquals(jobState.getProp("lower"), "case"); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertEquals(jobState.getProp("lower"), "case"); }
TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), "TestJob-1"); Assert.assertEquals(taskState.getStartTime(), this.startTime);
for (TaskState taskState : this.getTaskStates()) { taskExecutionInfos.add(taskState.toTaskExecutionInfo());
/** * Subclasses can override this method to do whatever processing on the {@link JobState} and its * associated {@link TaskState}s, e.g., aggregate task-level metrics into job-level metrics. */ protected void postProcessJobState(JobState jobState) { postProcessTaskStates(jobState.getTaskStates()); }
public Map<String, String> getMetadata(JobContext jobContext, EventName eventName) { List<TaskState> taskStates = jobContext.getJobState().getTaskStates(); switch (eventName) { case JOB_COMPLETE: return ImmutableMap.of(PROCESSED_COUNT_KEY, Long.toString(EventMetadataUtils.getProcessedCount(taskStates))); case JOB_FAILED: return ImmutableMap.of(MESSAGE_KEY, EventMetadataUtils.getTaskFailureExceptions(taskStates)); default: break; } return ImmutableMap.of(); } }
/** * Submits an event for each {@link TaskState} in the given {@link JobState}. */ private void submitTaskStateEvents(JobState jobState) { // Build Job Metadata applicable for TaskStates ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); Map<String, String> jobMetadata = jobMetadataBuilder.build(); // Submit event for each TaskState for (TaskState taskState : jobState.getTaskStates()) { submitTaskStateEvent(taskState, jobMetadata); } }
/** * Finalize the {@link JobState} before committing the job. */ void finalizeJobStateBeforeCommit() { this.jobState.setEndTime(System.currentTimeMillis()); this.jobState.setDuration(this.jobState.getEndTime() - this.jobState.getStartTime()); for (TaskState taskState : this.jobState.getTaskStates()) { // Set fork.branches explicitly here so the rest job flow can pick it up this.jobState .setProp(ConfigurationKeys.FORK_BRANCHES_KEY, taskState.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1)); } }