/** * Add a collection of {@link TaskState}s. * * @param taskStates collection of {@link TaskState}s to add */ public void addTaskStates(Collection<TaskState> taskStates) { for (TaskState taskState : taskStates) { this.taskStates.put(taskState.getTaskId(), taskState); } }
/** * Add a single {@link TaskState}. * * @param taskState {@link TaskState} to add */ public void addTaskState(TaskState taskState) { this.taskStates.put(taskState.getTaskId(), taskState); }
public void removeTaskState(TaskState taskState) { this.taskStates.remove(taskState.getTaskId()); this.taskCount--; }
public void addSkippedTaskState(TaskState taskState) { this.skippedTaskStates.put(taskState.getTaskId(), taskState); }
private static String name(TaskState taskState) { return "gobblin.metrics." + taskState.getJobId() + "." + taskState.getTaskId(); }
private static void cleanupStagingDataPerTask(JobState jobState) { Closer closer = Closer.create(); Map<String, ParallelRunner> parallelRunners = Maps.newHashMap(); try { for (TaskState taskState : jobState.getTaskStates()) { try { JobLauncherUtils.cleanTaskStagingData(taskState, LOG, closer, parallelRunners); } catch (IOException e) { LOG.error(String.format("Failed to clean staging data for task %s: %s", taskState.getTaskId(), e), e); } } } finally { try { closer.close(); } catch (IOException e) { LOG.error("Failed to clean staging data", e); } } }
/** * Submits an event for a given {@link TaskState}. It will include all metadata specified in the jobMetadata parameter. */ private void submitTaskStateEvent(TaskState taskState, Map<String, String> jobMetadata) { ImmutableMap.Builder<String, String> taskMetadataBuilder = new ImmutableMap.Builder<>(); taskMetadataBuilder.putAll(jobMetadata); taskMetadataBuilder.put(METADATA_TASK_ID, taskState.getTaskId()); taskMetadataBuilder.put(METADATA_TASK_START_TIME, Long.toString(taskState.getStartTime())); taskMetadataBuilder.put(METADATA_TASK_END_TIME, Long.toString(taskState.getEndTime())); taskMetadataBuilder.put(METADATA_TASK_WORKING_STATE, taskState.getWorkingState().toString()); taskMetadataBuilder.put(METADATA_TASK_FAILURE_CONTEXT, taskState.getTaskFailureException().or(UNKNOWN_VALUE)); taskMetadataBuilder.put(EventSubmitter.EVENT_TYPE, TASK_STATE); this.eventSubmitter.submit(TASK_STATE, taskMetadataBuilder.build()); } }
protected static List<Tag<?>> tagsForTask(TaskState taskState) { List<Tag<?>> tags = Lists.newArrayList(); tags.add(new Tag<>(TaskEvent.METADATA_TASK_ID, taskState.getTaskId())); tags.add(new Tag<>(TaskEvent.METADATA_TASK_ATTEMPT_ID, taskState.getTaskAttemptId().or(""))); tags.add(new Tag<>(ConfigurationKeys.DATASET_URN_KEY, taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN))); tags.addAll(getCustomTagsFromState(taskState)); return tags; }
/** * Convert this {@link TaskState} to a json document. * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} used to write the json document * @throws IOException */ public void toJson(JsonWriter jsonWriter, boolean keepConfig) throws IOException { jsonWriter.beginObject(); jsonWriter.name("task id").value(this.getTaskId()).name("task state").value(this.getWorkingState().name()) .name("start time").value(this.getStartTime()).name("end time").value(this.getEndTime()).name("duration") .value(this.getTaskDuration()).name("retry count") .value(this.getPropAsInt(ConfigurationKeys.TASK_RETRIES_KEY, 0)); // Also add failure exception information if it exists. This information is useful even in the // case that the task finally succeeds so we know what happened in the course of task execution. if (getTaskFailureException().isPresent()) { jsonWriter.name("exception").value(getTaskFailureException().get()); } if (keepConfig) { jsonWriter.name("properties"); jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); } jsonWriter.endObject(); }
public TaskIFaceWrapper(TaskIFace underlyingTask, TaskContext taskContext, CountDownLatch countDownLatch, TaskStateTracker taskStateTracker) { super(); this.underlyingTask = underlyingTask; this.taskContext = taskContext; this.jobId = taskContext.getTaskState().getJobId(); this.taskId = taskContext.getTaskState().getTaskId(); this.countDownLatch = countDownLatch; this.taskStateTracker = taskStateTracker; }
@Subscribe @Test(enabled = false) public void handleNewOutputTaskStateEvent(NewTaskCompletionEvent newOutputTaskStateEvent) { for (TaskState taskState : newOutputTaskStateEvent.getTaskStates()) { this.taskStateMap.put(taskState.getTaskId(), taskState); } } }
@Test(dependsOnMethods = "testPutIntoTaskStateStore") public void testCollectOutputTaskStates() throws Exception { this.taskStateCollectorService.runOneIteration(); Assert.assertEquals(this.jobState.getTaskStates().size(), 2); Assert.assertEquals(this.taskStateMap.size(), 2); Assert.assertEquals(this.taskStateMap.get(TASK_ID_0).getJobId(), JOB_ID); Assert.assertEquals(this.taskStateMap.get(TASK_ID_0).getTaskId(), TASK_ID_0); Assert.assertEquals(this.taskStateMap.get(TASK_ID_1).getJobId(), JOB_ID); Assert.assertEquals(this.taskStateMap.get(TASK_ID_1).getTaskId(), TASK_ID_1); }
@Override public Extractor<String, String> getExtractor(WorkUnitState workUnitState) { Extractor<String, String> extractor = super.getExtractor(workUnitState); if (((TaskState) workUnitState).getTaskId().endsWith("0")) { return new FaultyExtractor(workUnitState); } return extractor; } }
private void getTaskStateWithCommonAndSpecWuProps(int numTaskStates, DataInput in) throws IOException { Properties commonWuProps = new Properties(); for (int i = 0; i < numTaskStates; i++) { TaskState taskState = new TaskState(); taskState.readFields(in); if (i == 0) { commonWuProps.putAll(taskState.getWorkunit().getProperties()); } else { Properties newCommonWuProps = new Properties(); newCommonWuProps .putAll(Maps.difference(commonWuProps, taskState.getWorkunit().getProperties()).entriesInCommon()); commonWuProps = newCommonWuProps; } this.taskStates.put(taskState.getTaskId().intern(), taskState); } ImmutableProperties immutableCommonProperties = new ImmutableProperties(commonWuProps); for (TaskState taskState : this.taskStates.values()) { Properties newSpecProps = new Properties(); newSpecProps.putAll( Maps.difference(immutableCommonProperties, taskState.getWorkunit().getProperties()).entriesOnlyOnRight()); taskState.setWuProperties(immutableCommonProperties, newSpecProps); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = zkDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = this.fsDatasetStateStore.get(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = dbDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = this.fsDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test public void testSetAndGet() { this.taskState.setId("Task-1"); this.taskState.setHighWaterMark(2000); this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(this.startTime); this.taskState.setEndTime(this.startTime + 1000); this.taskState.setTaskDuration(1000); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); this.taskState.setProp("foo", "bar"); Assert.assertEquals(this.taskState.getJobId(), "Job-1"); Assert.assertEquals(this.taskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(this.taskState.getStartTime(), this.startTime); Assert.assertEquals(this.taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(this.taskState.getTaskDuration(), 1000); Assert.assertEquals(this.taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(this.taskState.getProp("foo"), "bar"); }