/** * Convert this {@link TaskState} to a json document. * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} used to write the json document * @throws IOException */ public void toJson(JsonWriter jsonWriter, boolean keepConfig) throws IOException { jsonWriter.beginObject(); jsonWriter.name("task id").value(this.getTaskId()).name("task state").value(this.getWorkingState().name()) .name("start time").value(this.getStartTime()).name("end time").value(this.getEndTime()).name("duration") .value(this.getTaskDuration()).name("retry count") .value(this.getPropAsInt(ConfigurationKeys.TASK_RETRIES_KEY, 0)); // Also add failure exception information if it exists. This information is useful even in the // case that the task finally succeeds so we know what happened in the course of task execution. if (getTaskFailureException().isPresent()) { jsonWriter.name("exception").value(getTaskFailureException().get()); } if (keepConfig) { jsonWriter.name("properties"); jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); } jsonWriter.endObject(); }
@Override public void onTaskCommitCompletion(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit())) { // Update record-level metrics after the task is done task.updateRecordMetrics(); task.updateByteMetrics(); } // Cancel the task state reporter associated with this task. The reporter might // not be found for the given task because the task fails before the task is // registered. So we need to make sure the reporter exists before calling cancel. if (this.scheduledReporters.containsKey(task.getTaskId())) { this.scheduledReporters.remove(task.getTaskId()).cancel(false); } LOGGER.info(String .format("Task %s completed in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState()));
@Test(dependsOnMethods = {"testSetAndGet"}) public void testSerDe() throws IOException { Closer closer = Closer.create(); try { ByteArrayOutputStream baos = closer.register(new ByteArrayOutputStream()); DataOutputStream dos = closer.register(new DataOutputStream(baos)); this.taskState.write(dos); ByteArrayInputStream bais = closer.register((new ByteArrayInputStream(baos.toByteArray()))); DataInputStream dis = closer.register((new DataInputStream(bais))); TaskState newTaskState = new TaskState(); newTaskState.readFields(dis); Assert.assertEquals(newTaskState.getJobId(), "Job-1"); Assert.assertEquals(newTaskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(newTaskState.getStartTime(), this.startTime); Assert.assertEquals(newTaskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(newTaskState.getTaskDuration(), 1000); Assert.assertEquals(newTaskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(newTaskState.getProp("foo"), "bar"); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
@Override public void onTaskCommitCompletion(Task task) { WorkUnit workUnit = task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { task.updateRecordMetrics(); task.updateByteMetrics(); if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(task); } } LOG.info(String .format("Task %s completed running in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
@Test public void testSetAndGet() { this.taskState.setId("Task-1"); this.taskState.setHighWaterMark(2000); this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(this.startTime); this.taskState.setEndTime(this.startTime + 1000); this.taskState.setTaskDuration(1000); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); this.taskState.setProp("foo", "bar"); Assert.assertEquals(this.taskState.getJobId(), "Job-1"); Assert.assertEquals(this.taskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(this.taskState.getStartTime(), this.startTime); Assert.assertEquals(this.taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(this.taskState.getTaskDuration(), 1000); Assert.assertEquals(this.taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(this.taskState.getProp("foo"), "bar"); }
Assert.assertEquals(taskState.getStartTime(), this.startTime); Assert.assertEquals(taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(taskState.getTaskDuration(), 1000); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertTrue(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY).startsWith("TestDataset"));
/** * Convert this {@link TaskState} to a json document. * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} used to write the json document * @throws IOException */ public void toJson(JsonWriter jsonWriter, boolean keepConfig) throws IOException { jsonWriter.beginObject(); jsonWriter.name("task id").value(this.getTaskId()).name("task state").value(this.getWorkingState().name()) .name("start time").value(this.getStartTime()).name("end time").value(this.getEndTime()).name("duration") .value(this.getTaskDuration()).name("retry count") .value(this.getPropAsInt(ConfigurationKeys.TASK_RETRIES_KEY, 0)); // Also add failure exception information if it exists. This information is useful even in the // case that the task finally succeeds so we know what happened in the course of task execution. if (getTaskFailureException().isPresent()) { jsonWriter.name("exception").value(getTaskFailureException().get()); } if (keepConfig) { jsonWriter.name("properties"); jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); } jsonWriter.endObject(); }
@Override public void onTaskCommitCompletion(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit())) { // Update record-level metrics after the task is done task.updateRecordMetrics(); task.updateByteMetrics(); } // Cancel the task state reporter associated with this task. The reporter might // not be found for the given task because the task fails before the task is // registered. So we need to make sure the reporter exists before calling cancel. if (this.scheduledReporters.containsKey(task.getTaskId())) { this.scheduledReporters.remove(task.getTaskId()).cancel(false); } LOGGER.info(String .format("Task %s completed in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState()));
@Override public void onTaskCommitCompletion(Task task) { WorkUnit workUnit = task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { task.updateRecordMetrics(); task.updateByteMetrics(); if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(task); } } LOG.info(String .format("Task %s completed running in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }