@Test public void testSetAndGet() { this.taskState.setId("Task-1"); this.taskState.setHighWaterMark(2000); this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(this.startTime); this.taskState.setEndTime(this.startTime + 1000); this.taskState.setTaskDuration(1000); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); this.taskState.setProp("foo", "bar"); Assert.assertEquals(this.taskState.getJobId(), "Job-1"); Assert.assertEquals(this.taskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(this.taskState.getStartTime(), this.startTime); Assert.assertEquals(this.taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(this.taskState.getTaskDuration(), 1000); Assert.assertEquals(this.taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(this.taskState.getProp("foo"), "bar"); }
private void getTaskStateWithCommonAndSpecWuProps(int numTaskStates, DataInput in) throws IOException { Properties commonWuProps = new Properties(); for (int i = 0; i < numTaskStates; i++) { TaskState taskState = new TaskState(); taskState.readFields(in); if (i == 0) { commonWuProps.putAll(taskState.getWorkunit().getProperties()); } else { Properties newCommonWuProps = new Properties(); newCommonWuProps .putAll(Maps.difference(commonWuProps, taskState.getWorkunit().getProperties()).entriesInCommon()); commonWuProps = newCommonWuProps; } this.taskStates.put(taskState.getTaskId().intern(), taskState); } ImmutableProperties immutableCommonProperties = new ImmutableProperties(commonWuProps); for (TaskState taskState : this.taskStates.values()) { Properties newSpecProps = new Properties(); newSpecProps.putAll( Maps.difference(immutableCommonProperties, taskState.getWorkunit().getProperties()).entriesOnlyOnRight()); taskState.setWuProperties(immutableCommonProperties, newSpecProps); } }
/** * Create the {@link Job} to run in this task. * @return the {@link Job} to run. If this method returns null, no job will be run and the task will be marked as successful. */ protected Job createJob() throws IOException { Job job = Job.getInstance(new Configuration()); for (Map.Entry<Object, Object> entry : this.taskContext.getTaskState().getProperties().entrySet()) { if (entry.getKey() instanceof String && ((String) entry.getKey()).startsWith(JOB_CONFIGURATION_PREFIX)) { String actualKey = ((String) entry.getKey()).substring(JOB_CONFIGURATION_PREFIX.length()); job.getConfiguration().set(actualKey, (String) entry.getValue()); } } return job; }
public TaskState(TaskState taskState) { super(taskState.getWorkunit(), taskState.getJobState(), taskState.getTaskBrokerNullable()); addAll(taskState); this.jobId = taskState.getProp(ConfigurationKeys.JOB_ID_KEY); this.taskId = taskState.getProp(ConfigurationKeys.TASK_ID_KEY); this.taskAttemptId = taskState.getTaskAttemptId(); this.setId(this.taskId); }
@SuppressWarnings("unchecked") private Class<? extends DataPublisher> getTaskPublisherClass() throws ReflectiveOperationException { if (this.taskState.contains(ConfigurationKeys.TASK_DATA_PUBLISHER_TYPE)) { return (Class<? extends DataPublisher>) Class .forName(this.taskState.getProp(ConfigurationKeys.TASK_DATA_PUBLISHER_TYPE)); } return (Class<? extends DataPublisher>) Class.forName( this.taskState.getProp(ConfigurationKeys.DATA_PUBLISHER_TYPE, ConfigurationKeys.DEFAULT_DATA_PUBLISHER_TYPE)); }
public void persistDatasetState(List<Dataset> datasets, List<LongWatermark> watermarks, String jobName) throws IOException { Preconditions.checkArgument(datasets.size() >= 2); for (int i = 0; i < datasets.size(); i++) { String datasetUrn = datasets.get(i).getUrn(); JobState.DatasetState datasetState = new JobState.DatasetState(jobName, TEST_JOB_ID); datasetState.setDatasetUrn(datasetUrn); datasetState.setState(JobState.RunningState.COMMITTED); datasetState.setId(datasetUrn); datasetState.setStartTime(this.startTime); datasetState.setEndTime(this.startTime + 1000); datasetState.setDuration(1000); TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); if (i < datasets.size() - 1) { taskState.setActualHighWatermark(watermarks.get(i)); } datasetState.addTaskState(taskState); this.fsDatasetStateStore.persistDatasetState(datasetUrn, datasetState); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = zkDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = {"testSetAndGet"}) public void testSerDe() throws IOException { Closer closer = Closer.create(); try { ByteArrayOutputStream baos = closer.register(new ByteArrayOutputStream()); DataOutputStream dos = closer.register(new DataOutputStream(baos)); this.taskState.write(dos); ByteArrayInputStream bais = closer.register((new ByteArrayInputStream(baos.toByteArray()))); DataInputStream dis = closer.register((new DataInputStream(bais))); TaskState newTaskState = new TaskState(); newTaskState.readFields(dis); Assert.assertEquals(newTaskState.getJobId(), "Job-1"); Assert.assertEquals(newTaskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(newTaskState.getStartTime(), this.startTime); Assert.assertEquals(newTaskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(newTaskState.getTaskDuration(), 1000); Assert.assertEquals(newTaskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(newTaskState.getProp("foo"), "bar"); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
private TaskState getStreamingTaskState() { WorkUnitState workUnitState = new WorkUnitState(WorkUnit.create( new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName()))); workUnitState.setProp(ConfigurationKeys.TASK_KEY_KEY, "1234"); TaskState taskState = new TaskState(workUnitState); taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false)); taskState.setProp(TaskConfigurationKeys.TASK_EXECUTION_MODE, ExecutionModel.STREAMING.name()); taskState.setJobId("1234"); taskState.setTaskId("testContinuousTaskId"); return taskState; }
/** * Convert this {@link TaskState} to a json document. * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} used to write the json document * @throws IOException */ public void toJson(JsonWriter jsonWriter, boolean keepConfig) throws IOException { jsonWriter.beginObject(); jsonWriter.name("task id").value(this.getTaskId()).name("task state").value(this.getWorkingState().name()) .name("start time").value(this.getStartTime()).name("end time").value(this.getEndTime()).name("duration") .value(this.getTaskDuration()).name("retry count") .value(this.getPropAsInt(ConfigurationKeys.TASK_RETRIES_KEY, 0)); // Also add failure exception information if it exists. This information is useful even in the // case that the task finally succeeds so we know what happened in the course of task execution. if (getTaskFailureException().isPresent()) { jsonWriter.name("exception").value(getTaskFailureException().get()); } if (keepConfig) { jsonWriter.name("properties"); jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); } jsonWriter.endObject(); }
workUnitState.setProp(ConfigurationKeys.TASK_ID_KEY, "TestTask-" + i); workUnitState.setProp(ConfigurationKeys.DATASET_URN_KEY, "TestDataset" + i); TaskState taskState = new TaskState(workUnitState); taskState.setTaskId("TestTask-" + i); taskState.setId(taskState.getTaskId()); taskState.setStartTime(this.startTime); taskState.setEndTime(this.startTime + 1000); taskState.setTaskDuration(1000); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); taskState.setProp("foo", "bar");
for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), "TestJob-1"); Assert.assertEquals(taskState.getStartTime(), this.startTime); Assert.assertEquals(taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(taskState.getTaskDuration(), 1000); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertTrue(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY).startsWith("TestDataset")); Assert.assertEquals(taskState.getProp("foo"), "bar"); if (initial) { Assert.assertEquals(taskState.getWorkunit().getCommonProperties().size(), 0); Assert.assertEquals(taskState.getWorkunit().getSpecProperties().size(), 4); Assert.assertEquals(taskState.getProp("common1"), "1"); Assert.assertEquals(taskState.getProp("common2"), "2"); Assert.assertEquals(taskState.getProp("spec1"), String.valueOf(i)); Assert.assertEquals(taskState.getProp("spec2"), "spec" + i); } else { Assert.assertEquals(taskState.getWorkunit().getCommonProperties().size(), 2); Assert.assertEquals(taskState.getWorkunit().getCommonProperties().getProperty("common1"), "1"); Assert.assertEquals(taskState.getWorkunit().getCommonProperties().getProperty("common2"), "2"); Assert.assertEquals(taskState.getWorkunit().getSpecProperties().size(), 2); Assert.assertEquals(taskState.getWorkunit().getSpecProperties().getProperty("spec1"), String.valueOf(i)); Assert.assertEquals(taskState.getWorkunit().getSpecProperties().getProperty("spec2"), "spec" + i); Assert.assertEquals(taskState.getProp("common1"), "1"); Assert.assertEquals(taskState.getProp("common2"), "2"); Assert.assertEquals(taskState.getProp("spec1"), String.valueOf(i)); Assert.assertEquals(taskState.getProp("spec2"), "spec" + i); taskStateIds.add(taskState.getTaskId());
@Override public void onTaskCommitCompletion(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit())) { // Update record-level metrics after the task is done task.updateRecordMetrics(); task.updateByteMetrics(); } // Cancel the task state reporter associated with this task. The reporter might // not be found for the given task because the task fails before the task is // registered. So we need to make sure the reporter exists before calling cancel. if (this.scheduledReporters.containsKey(task.getTaskId())) { this.scheduledReporters.remove(task.getTaskId()).cancel(false); } LOGGER.info(String .format("Task %s completed in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
TaskState taskState = mock (TaskState.class); WorkUnit workUnit = mock (WorkUnit.class); Mockito.when(taskState.getWorkunit()).thenReturn(workUnit); Mockito.when(taskState.getJobId()).thenReturn("123"); Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555")); Mockito.when(taskState.getTaskId()).thenReturn("888"); Mockito.when(limiter.acquirePermits(1)).thenReturn(null); Mockito.when (taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN)).thenReturn("file://xyz"); Mockito.when(workUnit.getProperties()).thenReturn(properties); LimitingExtractorDecorator<String, String> decorator = new LimitingExtractorDecorator<>(extractor, limiter, taskState);
this.taskContext = context; this.taskState = context.getTaskState(); this.jobId = this.taskState.getJobId(); this.taskId = this.taskState.getTaskId(); this.taskKey = this.taskState.getTaskKey(); this.isIgnoreCloseFailures = this.taskState.getJobState().getPropAsBoolean(ConfigurationKeys.TASK_IGNORE_CLOSE_FAILURES, false); this.taskStateTracker = taskStateTracker; this.taskExecutor = taskExecutor; this.countDownLatch = countDownLatch; this.closer = Closer.create(); this.closer.register(this.taskState.getTaskBrokerNullable()); this.extractor = closer.register(new InstrumentedExtractorDecorator<>(this.taskState, this.taskContext.getExtractor())); Config config; try { config = ConfigUtils.propertiesToConfig(taskState.getProperties()); } catch (Exception e) { LOG.warn("Failed to deserialize taskState into Config.. continuing with an empty config", e);
/** * Submits an event for a given {@link TaskState}. It will include all metadata specified in the jobMetadata parameter. */ private void submitTaskStateEvent(TaskState taskState, Map<String, String> jobMetadata) { ImmutableMap.Builder<String, String> taskMetadataBuilder = new ImmutableMap.Builder<>(); taskMetadataBuilder.putAll(jobMetadata); taskMetadataBuilder.put(METADATA_TASK_ID, taskState.getTaskId()); taskMetadataBuilder.put(METADATA_TASK_START_TIME, Long.toString(taskState.getStartTime())); taskMetadataBuilder.put(METADATA_TASK_END_TIME, Long.toString(taskState.getEndTime())); taskMetadataBuilder.put(METADATA_TASK_WORKING_STATE, taskState.getWorkingState().toString()); taskMetadataBuilder.put(METADATA_TASK_FAILURE_CONTEXT, taskState.getTaskFailureException().or(UNKNOWN_VALUE)); taskMetadataBuilder.put(EventSubmitter.EVENT_TYPE, TASK_STATE); this.eventSubmitter.submit(TASK_STATE, taskMetadataBuilder.build()); } }
@Override public TaskState getTaskState() { TaskState taskState = this.taskContext.getTaskState(); taskState.setTaskId(getTaskId()); taskState.setJobId(getJobId()); taskState.setWorkingState(getWorkingState()); taskState.addAll(getExecutionMetadata()); taskState.addAll(getPersistentState()); return taskState; }
JobContext jobContext = Mockito.mock(JobContext.class); JobState jobState = new JobState("jobName", "1234"); TaskState taskState1 = new TaskState(); TaskState taskState2 = new TaskState(); taskState1.setTaskId("1"); taskState1.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, "exception1"); taskState2.setTaskId("2"); taskState2.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, "exception2"); taskState2.setProp(EventMetadataUtils.TASK_FAILURE_MESSAGE_KEY, "failureMessage2");
@Test public void testPutIntoTaskStateStore() throws IOException { TaskState taskState1 = new TaskState(); taskState1.setJobId(JOB_ID); taskState1.setTaskId(TASK_ID_0); this.taskStateStore.put(JOB_ID, TASK_ID_0 + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, taskState1); TaskState taskState2 = new TaskState(); taskState2.setJobId(JOB_ID); taskState2.setTaskId(TASK_ID_1); this.taskStateStore.put(JOB_ID, TASK_ID_1 + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, taskState2); }