/** * Mark the completion of this {@link Task}. */ public void markTaskCompletion() { if (this.countDownLatch.isPresent()) { this.countDownLatch.get().countDown(); } this.taskState.setProp(ConfigurationKeys.TASK_RETRIES_KEY, this.retryCount.get()); }
/** * If not already present, set the {@link ConfigurationKeys#TASK_FAILURE_EXCEPTION_KEY} to a {@link String} * representation of the given {@link Throwable}. */ public void setTaskFailureException(Throwable taskFailureException) { if (!this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { this.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, Throwables.getStackTraceAsString(taskFailureException)); } }
@Override public void close() throws IOException { // Tell this fork that the parent task is done. This is a second chance call if the parent // task failed and didn't do so through the normal way of calling markParentTaskDone(). this.parentTaskDone = true; // Record the fork state into the task state that will be persisted into the state store this.taskState.setProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.FORK_STATE_KEY, this.branches, this.index), this.forkState.get().name()); try { this.closer.close(); } finally { if (this.writer.isPresent()) { this.writer.get().cleanup(); } } }
forkTaskState.setProp(ConfigurationKeys.FORK_BRANCH_ID_KEY, index);
forkTaskState.setProp(ConfigurationKeys.FORK_BRANCH_ID_KEY, index);
public TaskContext(WorkUnitState workUnitState) { this.taskState = new TaskState(workUnitState); this.taskMetrics = TaskMetrics.get(this.taskState); this.taskState.setProp(Instrumented.METRIC_CONTEXT_NAME_KEY, this.taskMetrics.getName()); }
private void failTask(Throwable t) { LOG.error(String.format("Task %s failed", this.taskId), t); this.taskState.setWorkingState(WorkUnitState.WorkingState.FAILED); this.taskState.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, Throwables.getStackTraceAsString(t)); // Send task failure event FailureEventBuilder failureEvent = new FailureEventBuilder(FAILED_TASK_EVENT); failureEvent.setRootCause(t); failureEvent.addMetadata(TASK_STATE, this.taskState.toString()); failureEvent.submit(taskContext.getTaskMetrics().getMetricContext()); }
private TaskState getStreamingTaskState() { WorkUnitState workUnitState = new WorkUnitState(WorkUnit.create( new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName()))); workUnitState.setProp(ConfigurationKeys.TASK_KEY_KEY, "1234"); TaskState taskState = new TaskState(workUnitState); taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false)); taskState.setProp(TaskConfigurationKeys.TASK_EXECUTION_MODE, ExecutionModel.STREAMING.name()); taskState.setJobId("1234"); taskState.setTaskId("testContinuousTaskId"); return taskState; }
/** * Test that forks work correctly when the operator picks all outgoing forks */ @Test(dataProvider = "stateOverrides") public void testForkCorrectnessIdentity(State overrides) throws Exception { // Create a TaskState TaskState taskState = getEmptyTestTaskState("testForkTaskId"); taskState.addAll(overrides); int numRecords = 100; int numForks = 5; // Identity Fork Operator looks for number of forks in work unit state. taskState.setProp(ConfigurationKeys.FORK_BRANCHES_KEY, "" + numForks); ForkOperator mockForkOperator = new IdentityForkOperator(); ArrayList<ArrayList<Object>> recordCollectors = runTaskAndGetResults(taskState, numRecords, numForks, mockForkOperator); // Check that we got the right records in the collectors int recordsPerFork = numRecords; for (int forkNumber=0; forkNumber < numForks; ++ forkNumber) { ArrayList<Object> forkRecords = recordCollectors.get(forkNumber); Assert.assertEquals(forkRecords.size(), recordsPerFork); for (int j=0; j < recordsPerFork; ++j) { Object forkRecord = forkRecords.get(j); Assert.assertEquals((String) forkRecord, "" + j); } } }
TaskState getEmptyTestTaskState(String taskId) { // Create a TaskState WorkUnit workUnit = WorkUnit.create( new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName())); workUnit.setProp(ConfigurationKeys.TASK_KEY_KEY, "taskKey"); TaskState taskState = new TaskState(new WorkUnitState(workUnit)); taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false)); taskState.setTaskId(taskId); taskState.setJobId("1234"); return taskState; }
@Test public void testPersistWatermarkStateToZk() throws IOException { CheckpointableWatermark watermark = new DefaultCheckpointableWatermark("source", new LongWatermark(startTime)); TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setProp(ConfigurationKeys.JOB_NAME_KEY, "JobName-" + startTime); // watermark storage configuration taskState.setProp(StateStoreBasedWatermarkStorage.WATERMARK_STORAGE_TYPE_KEY, "zk"); taskState.setProp(StateStoreBasedWatermarkStorage.WATERMARK_STORAGE_CONFIG_PREFIX + ZkStateStoreConfigurationKeys.STATE_STORE_ZK_CONNECT_STRING_KEY, testingServer.getConnectString()); StateStoreBasedWatermarkStorage watermarkStorage = new StateStoreBasedWatermarkStorage(taskState); watermarkStorage.commitWatermarks(ImmutableList.of(watermark)); Map<String, CheckpointableWatermark> watermarkMap = watermarkStorage.getCommittedWatermarks(DefaultCheckpointableWatermark.class, ImmutableList.of("source")); Assert.assertEquals(watermarkMap.size(), 1); Assert.assertEquals(((LongWatermark) watermarkMap.get("source").getWatermark()).getValue(), startTime); }
TaskState getEmptyTestTaskState(String taskId) { // Create a TaskState WorkUnit workUnit = WorkUnit.create( new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName())); workUnit.setProp(ConfigurationKeys.TASK_KEY_KEY, "taskKey"); TaskState taskState = new TaskState(new WorkUnitState(workUnit)); taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false)); taskState.setTaskId(taskId); taskState.setJobId("1234"); return taskState; }
taskState.setProp(ConfigurationKeys.TASK_SYNCHRONOUS_EXECUTION_MODEL_KEY, false);
@Test public void testProcessedCount() throws Exception { JobContext jobContext = Mockito.mock(JobContext.class); JobState jobState = new JobState("jobName", "1234"); TaskState taskState1 = new TaskState(); TaskState taskState2 = new TaskState(); taskState1.setTaskId("1"); taskState1.setProp(ConfigurationKeys.WRITER_RECORDS_WRITTEN, "1"); taskState2.setTaskId("2"); taskState2.setProp(ConfigurationKeys.WRITER_RECORDS_WRITTEN, "22"); jobState.addTaskState(taskState1); jobState.addTaskState(taskState2); Mockito.when(jobContext.getJobState()).thenReturn(jobState); ClusterEventMetadataGenerator metadataGenerator = new ClusterEventMetadataGenerator(); Map<String, String> metadata; // processed count is not in job cancel event metadata = metadataGenerator.getMetadata(jobContext, EventName.JOB_CANCEL); Assert.assertEquals(metadata.get("processedCount"), null); // processed count is in job complete event metadata = metadataGenerator.getMetadata(jobContext, EventName.getEnumFromEventId("JobCompleteTimer")); Assert.assertEquals(metadata.get("processedCount"), "23"); }
taskState.setProp(ConfigurationKeys.TASK_SYNCHRONOUS_EXECUTION_MODEL_KEY, false);
@Test public void testSetAndGet() { this.taskState.setId("Task-1"); this.taskState.setHighWaterMark(2000); this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(this.startTime); this.taskState.setEndTime(this.startTime + 1000); this.taskState.setTaskDuration(1000); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); this.taskState.setProp("foo", "bar"); Assert.assertEquals(this.taskState.getJobId(), "Job-1"); Assert.assertEquals(this.taskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(this.taskState.getStartTime(), this.startTime); Assert.assertEquals(this.taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(this.taskState.getTaskDuration(), 1000); Assert.assertEquals(this.taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(this.taskState.getProp("foo"), "bar"); }
taskState.setJobId("job1_id2"); taskState.setTaskId("task123"); taskState.setProp("key", "value"); dataset2State.addTaskState(taskState);
taskState.setTaskDuration(1000); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); taskState.setProp("foo", "bar");