if (workUnitState.getWorkingState().equals(WorkUnitState.WorkingState.COMMITTED))
taskExecutionInfo.setState(TaskStateEnum.valueOf(getWorkingState().name())); if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY));
taskExecutionInfo.setState(TaskStateEnum.valueOf(getWorkingState().name())); if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY));
if (workUnitState.getWorkingState().equals(WorkUnitState.WorkingState.COMMITTED))
/** * Convert this {@link TaskState} to a json document. * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} used to write the json document * @throws IOException */ public void toJson(JsonWriter jsonWriter, boolean keepConfig) throws IOException { jsonWriter.beginObject(); jsonWriter.name("task id").value(this.getTaskId()).name("task state").value(this.getWorkingState().name()) .name("start time").value(this.getStartTime()).name("end time").value(this.getEndTime()).name("duration") .value(this.getTaskDuration()).name("retry count") .value(this.getPropAsInt(ConfigurationKeys.TASK_RETRIES_KEY, 0)); // Also add failure exception information if it exists. This information is useful even in the // case that the task finally succeeds so we know what happened in the course of task execution. if (getTaskFailureException().isPresent()) { jsonWriter.name("exception").value(getTaskFailureException().get()); } if (keepConfig) { jsonWriter.name("properties"); jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); } jsonWriter.endObject(); }
/** * Gets the LWM for this job runs. The new LWM is the HWM of the previous run + 1 unit (day,hour,minute..etc). * If there was no previous execution then it is set to the given lowWaterMark + 1 unit. */ private long getLowWaterMark(Iterable<WorkUnitState> previousStates, String lowWaterMark) { long lowWaterMarkValue = retriever.getWatermarkFromString(lowWaterMark); // Find the max HWM from the previous states, this is the new current LWM for (WorkUnitState previousState : previousStates) { if (previousState.getWorkingState().equals(WorkUnitState.WorkingState.COMMITTED)) { long previousHighWaterMark = previousState.getWorkunit().getHighWaterMark(); if (previousHighWaterMark > lowWaterMarkValue) { lowWaterMarkValue = previousHighWaterMark; } } } return lowWaterMarkValue + getRetriever().getWatermarkIncrementMs(); }
/** * Submits an event for a given {@link TaskState}. It will include all metadata specified in the jobMetadata parameter. */ private void submitTaskStateEvent(TaskState taskState, Map<String, String> jobMetadata) { ImmutableMap.Builder<String, String> taskMetadataBuilder = new ImmutableMap.Builder<>(); taskMetadataBuilder.putAll(jobMetadata); taskMetadataBuilder.put(METADATA_TASK_ID, taskState.getTaskId()); taskMetadataBuilder.put(METADATA_TASK_START_TIME, Long.toString(taskState.getStartTime())); taskMetadataBuilder.put(METADATA_TASK_END_TIME, Long.toString(taskState.getEndTime())); taskMetadataBuilder.put(METADATA_TASK_WORKING_STATE, taskState.getWorkingState().toString()); taskMetadataBuilder.put(METADATA_TASK_FAILURE_CONTEXT, taskState.getTaskFailureException().or(UNKNOWN_VALUE)); taskMetadataBuilder.put(EventSubmitter.EVENT_TYPE, TASK_STATE); this.eventSubmitter.submit(TASK_STATE, taskMetadataBuilder.build()); } }
@Override public String getProgress() { return getWorkingState().toString(); }
@Override public boolean apply(WorkUnitState workUnitState) { return workUnitState.getWorkingState().equals(WorkingState.SUCCESSFUL); } }
@Override public boolean apply(WorkUnitState input) { return null == input || !WorkingState.SUCCESSFUL.equals(input.getWorkingState()); } };
/** * Set the current runtime state of the {@link WorkUnit}. * * @param state {@link WorkingState} of the {@link WorkUnit} */ public void setWorkingState(WorkingState state) { setProp(ConfigurationKeys.WORK_UNIT_WORKING_STATE_KEY, state.toString()); }
/** * Convert this {@link TaskState} to a json document. * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} used to write the json document * @throws IOException */ public void toJson(JsonWriter jsonWriter, boolean keepConfig) throws IOException { jsonWriter.beginObject(); jsonWriter.name("task id").value(this.getTaskId()).name("task state").value(this.getWorkingState().name()) .name("start time").value(this.getStartTime()).name("end time").value(this.getEndTime()).name("duration") .value(this.getTaskDuration()).name("retry count") .value(this.getPropAsInt(ConfigurationKeys.TASK_RETRIES_KEY, 0)); // Also add failure exception information if it exists. This information is useful even in the // case that the task finally succeeds so we know what happened in the course of task execution. if (getTaskFailureException().isPresent()) { jsonWriter.name("exception").value(getTaskFailureException().get()); } if (keepConfig) { jsonWriter.name("properties"); jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); } jsonWriter.endObject(); }
@Test(dependsOnMethods = {"testSetAndGet"}) public void testToJobExecutionInfo() { JobExecutionInfo jobExecutionInfo = this.jobState.toJobExecutionInfo(); Assert.assertEquals(jobExecutionInfo.getJobName(), "TestJob"); Assert.assertEquals(jobExecutionInfo.getJobId(), "TestJob-1"); Assert.assertEquals(jobExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(jobExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(jobExecutionInfo.getDuration().longValue(), 1000L); Assert.assertEquals(jobExecutionInfo.getState().name(), JobState.RunningState.COMMITTED.name()); Assert.assertEquals(jobExecutionInfo.getLaunchedTasks().intValue(), 3); Assert.assertEquals(jobExecutionInfo.getCompletedTasks().intValue(), 3); Assert.assertEquals(jobExecutionInfo.getJobProperties().get("foo"), "bar"); List<String> taskStateIds = Lists.newArrayList(); for (TaskExecutionInfo taskExecutionInfo : jobExecutionInfo.getTaskExecutions()) { Assert.assertEquals(taskExecutionInfo.getJobId(), "TestJob-1"); Assert.assertEquals(taskExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(taskExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(taskExecutionInfo.getDuration().longValue(), 1000); Assert.assertEquals(taskExecutionInfo.getState().name(), WorkUnitState.WorkingState.COMMITTED.name()); Assert.assertEquals(taskExecutionInfo.getTaskProperties().get("foo"), "bar"); taskStateIds.add(taskExecutionInfo.getTaskId()); } Collections.sort(taskStateIds); Assert.assertEquals(taskStateIds, Lists.newArrayList("TestTask-0", "TestTask-1", "TestTask-2")); } }
/** * Gets the LWM for this job runs. The new LWM is the HWM of the previous run + 1 unit (day,hour,minute..etc). * If there was no previous execution then it is set to the given lowWaterMark + 1 unit. */ private long getLowWaterMark(Iterable<WorkUnitState> previousStates, String lowWaterMark) { long lowWaterMarkValue = retriever.getWatermarkFromString(lowWaterMark); // Find the max HWM from the previous states, this is the new current LWM for (WorkUnitState previousState : previousStates) { if (previousState.getWorkingState().equals(WorkUnitState.WorkingState.COMMITTED)) { long previousHighWaterMark = previousState.getWorkunit().getHighWaterMark(); if (previousHighWaterMark > lowWaterMarkValue) { lowWaterMarkValue = previousHighWaterMark; } } } return lowWaterMarkValue + getRetriever().getWatermarkIncrementMs(); }
/** * Submits an event for a given {@link TaskState}. It will include all metadata specified in the jobMetadata parameter. */ private void submitTaskStateEvent(TaskState taskState, Map<String, String> jobMetadata) { ImmutableMap.Builder<String, String> taskMetadataBuilder = new ImmutableMap.Builder<>(); taskMetadataBuilder.putAll(jobMetadata); taskMetadataBuilder.put(METADATA_TASK_ID, taskState.getTaskId()); taskMetadataBuilder.put(METADATA_TASK_START_TIME, Long.toString(taskState.getStartTime())); taskMetadataBuilder.put(METADATA_TASK_END_TIME, Long.toString(taskState.getEndTime())); taskMetadataBuilder.put(METADATA_TASK_WORKING_STATE, taskState.getWorkingState().toString()); taskMetadataBuilder.put(METADATA_TASK_FAILURE_CONTEXT, taskState.getTaskFailureException().or(UNKNOWN_VALUE)); taskMetadataBuilder.put(EventSubmitter.EVENT_TYPE, TASK_STATE); this.eventSubmitter.submit(TASK_STATE, taskMetadataBuilder.build()); } }
@Override public String getProgress() { return getWorkingState().toString(); }
@Test(dependsOnMethods = {"testSetAndGet"}) public void testToTaskExecutionInfo() { TaskExecutionInfo taskExecutionInfo = this.taskState.toTaskExecutionInfo(); Assert.assertEquals(taskExecutionInfo.getJobId(), "Job-1"); Assert.assertEquals(taskExecutionInfo.getTaskId(), "Task-1"); Assert.assertEquals(taskExecutionInfo.getHighWatermark().longValue(), 2000L); Assert.assertEquals(taskExecutionInfo.getStartTime().longValue(), this.startTime); Assert.assertEquals(taskExecutionInfo.getEndTime().longValue(), this.startTime + 1000); Assert.assertEquals(taskExecutionInfo.getDuration().longValue(), 1000L); Assert.assertEquals(taskExecutionInfo.getState().name(), WorkUnitState.WorkingState.COMMITTED.name()); Assert.assertEquals(taskExecutionInfo.getTaskProperties().get("foo"), "bar"); } }
/** * Set the current runtime state of the {@link WorkUnit}. * * @param state {@link WorkingState} of the {@link WorkUnit} */ public void setWorkingState(WorkingState state) { setProp(ConfigurationKeys.WORK_UNIT_WORKING_STATE_KEY, state.toString()); }