/** * Submits an event for a given {@link TaskState}. It will include all metadata specified in the jobMetadata parameter. */ private void submitTaskStateEvent(TaskState taskState, Map<String, String> jobMetadata) { ImmutableMap.Builder<String, String> taskMetadataBuilder = new ImmutableMap.Builder<>(); taskMetadataBuilder.putAll(jobMetadata); taskMetadataBuilder.put(METADATA_TASK_ID, taskState.getTaskId()); taskMetadataBuilder.put(METADATA_TASK_START_TIME, Long.toString(taskState.getStartTime())); taskMetadataBuilder.put(METADATA_TASK_END_TIME, Long.toString(taskState.getEndTime())); taskMetadataBuilder.put(METADATA_TASK_WORKING_STATE, taskState.getWorkingState().toString()); taskMetadataBuilder.put(METADATA_TASK_FAILURE_CONTEXT, taskState.getTaskFailureException().or(UNKNOWN_VALUE)); taskMetadataBuilder.put(EventSubmitter.EVENT_TYPE, TASK_STATE); this.eventSubmitter.submit(TASK_STATE, taskMetadataBuilder.build()); } }
/** * Convert this {@link TaskState} to a json document. * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} used to write the json document * @throws IOException */ public void toJson(JsonWriter jsonWriter, boolean keepConfig) throws IOException { jsonWriter.beginObject(); jsonWriter.name("task id").value(this.getTaskId()).name("task state").value(this.getWorkingState().name()) .name("start time").value(this.getStartTime()).name("end time").value(this.getEndTime()).name("duration") .value(this.getTaskDuration()).name("retry count") .value(this.getPropAsInt(ConfigurationKeys.TASK_RETRIES_KEY, 0)); // Also add failure exception information if it exists. This information is useful even in the // case that the task finally succeeds so we know what happened in the course of task execution. if (getTaskFailureException().isPresent()) { jsonWriter.name("exception").value(getTaskFailureException().get()); } if (keepConfig) { jsonWriter.name("properties"); jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); } jsonWriter.endObject(); }
private TaskState createMockTaskState() { TaskState taskState = mock(TaskState.class, Mockito.RETURNS_SMART_NULLS); when(taskState.getWorkingState()).thenReturn(WorkUnitState.WorkingState.SUCCESSFUL); when(taskState.getTaskFailureException()).thenReturn(Optional.<String> absent()); return taskState; } }
private void finalizeDatasetState(JobState.DatasetState datasetState, String datasetUrn) { for (TaskState taskState : datasetState.getTaskStates()) { // Backoff the actual high watermark to the low watermark for each task that has not been committed if (taskState.getWorkingState() != WorkUnitState.WorkingState.COMMITTED) { taskState.backoffActualHighWatermark(); if (this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS) { // Determine the final dataset state based on the task states (post commit) and the job commit policy. // 1. If COMMIT_ON_FULL_SUCCESS is used, the processing of the dataset is considered failed if any // task for the dataset failed to be committed. // 2. Otherwise, the processing of the dataset is considered successful even if some tasks for the // dataset failed to be committed. datasetState.setState(JobState.RunningState.FAILED); Optional<String> taskStateException = taskState.getTaskFailureException(); log.warn("At least one task did not committed successfully. Setting dataset state to FAILED.", taskStateException.isPresent() ? taskStateException.get() : "Exception not set."); } } } datasetState.setId(datasetUrn); }
/** * Submits an event for a given {@link TaskState}. It will include all metadata specified in the jobMetadata parameter. */ private void submitTaskStateEvent(TaskState taskState, Map<String, String> jobMetadata) { ImmutableMap.Builder<String, String> taskMetadataBuilder = new ImmutableMap.Builder<>(); taskMetadataBuilder.putAll(jobMetadata); taskMetadataBuilder.put(METADATA_TASK_ID, taskState.getTaskId()); taskMetadataBuilder.put(METADATA_TASK_START_TIME, Long.toString(taskState.getStartTime())); taskMetadataBuilder.put(METADATA_TASK_END_TIME, Long.toString(taskState.getEndTime())); taskMetadataBuilder.put(METADATA_TASK_WORKING_STATE, taskState.getWorkingState().toString()); taskMetadataBuilder.put(METADATA_TASK_FAILURE_CONTEXT, taskState.getTaskFailureException().or(UNKNOWN_VALUE)); taskMetadataBuilder.put(EventSubmitter.EVENT_TYPE, TASK_STATE); this.eventSubmitter.submit(TASK_STATE, taskMetadataBuilder.build()); } }
/** * Convert this {@link TaskState} to a json document. * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} used to write the json document * @throws IOException */ public void toJson(JsonWriter jsonWriter, boolean keepConfig) throws IOException { jsonWriter.beginObject(); jsonWriter.name("task id").value(this.getTaskId()).name("task state").value(this.getWorkingState().name()) .name("start time").value(this.getStartTime()).name("end time").value(this.getEndTime()).name("duration") .value(this.getTaskDuration()).name("retry count") .value(this.getPropAsInt(ConfigurationKeys.TASK_RETRIES_KEY, 0)); // Also add failure exception information if it exists. This information is useful even in the // case that the task finally succeeds so we know what happened in the course of task execution. if (getTaskFailureException().isPresent()) { jsonWriter.name("exception").value(getTaskFailureException().get()); } if (keepConfig) { jsonWriter.name("properties"); jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); } jsonWriter.endObject(); }
private void finalizeDatasetState(JobState.DatasetState datasetState, String datasetUrn) { for (TaskState taskState : datasetState.getTaskStates()) { // Backoff the actual high watermark to the low watermark for each task that has not been committed if (taskState.getWorkingState() != WorkUnitState.WorkingState.COMMITTED) { taskState.backoffActualHighWatermark(); if (this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS) { // Determine the final dataset state based on the task states (post commit) and the job commit policy. // 1. If COMMIT_ON_FULL_SUCCESS is used, the processing of the dataset is considered failed if any // task for the dataset failed to be committed. // 2. Otherwise, the processing of the dataset is considered successful even if some tasks for the // dataset failed to be committed. datasetState.setState(JobState.RunningState.FAILED); Optional<String> taskStateException = taskState.getTaskFailureException(); log.warn("At least one task did not committed successfully. Setting dataset state to FAILED.", taskStateException.isPresent() ? taskStateException.get() : "Exception not set."); } } } datasetState.setId(datasetUrn); }