/** * Get task states of {@link Task}s of this job as {@link WorkUnitState}s. * * @return a list of {@link WorkUnitState}s */ public List<WorkUnitState> getTaskStatesAsWorkUnitStates() { ImmutableList.Builder<WorkUnitState> builder = ImmutableList.builder(); for (TaskState taskState : this.taskStates.values()) { WorkUnitState workUnitState = new WorkUnitState(taskState.getWorkunit(), taskState.getJobState()); workUnitState.setId(taskState.getId()); workUnitState.addAll(taskState); builder.add(workUnitState); } return builder.build(); }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = zkDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = this.fsDatasetStateStore.get(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = dbDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = this.fsDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
private void testDatasetStates(List<Dataset> datasets, List<LongWatermark> watermarks, String jobName) throws IOException { Preconditions.checkArgument(datasets.size() >= 2); for (int i = 0; i < datasets.size(); i++) { JobState.DatasetState datasetState = this.fsDatasetStateStore.getLatestDatasetState(jobName, datasets.get(i).getUrn()); Assert.assertEquals(datasetState.getDatasetUrn(), datasets.get(i).getUrn()); Assert.assertEquals(datasetState.getJobName(), jobName); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 1); TaskState taskState = datasetState.getTaskStates().get(0); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); if (i < datasets.size() - 1) { Assert.assertEquals(taskState.getActualHighWatermark(LongWatermark.class).getValue(), watermarks.get(i).getValue()); } } }
@Test public void testSetAndGet() { this.taskState.setId("Task-1"); this.taskState.setHighWaterMark(2000); this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(this.startTime); this.taskState.setEndTime(this.startTime + 1000); this.taskState.setTaskDuration(1000); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); this.taskState.setProp("foo", "bar"); Assert.assertEquals(this.taskState.getJobId(), "Job-1"); Assert.assertEquals(this.taskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(this.taskState.getStartTime(), this.startTime); Assert.assertEquals(this.taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(this.taskState.getTaskDuration(), 1000); Assert.assertEquals(this.taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(this.taskState.getProp("foo"), "bar"); }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertNotEquals(jobState.getProp("lower"), "case"); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertEquals(jobState.getProp("lower"), "case"); }
/** * Get task states of {@link Task}s of this job as {@link WorkUnitState}s. * * @return a list of {@link WorkUnitState}s */ public List<WorkUnitState> getTaskStatesAsWorkUnitStates() { ImmutableList.Builder<WorkUnitState> builder = ImmutableList.builder(); for (TaskState taskState : this.taskStates.values()) { WorkUnitState workUnitState = new WorkUnitState(taskState.getWorkunit(), taskState.getJobState()); workUnitState.setId(taskState.getId()); workUnitState.addAll(taskState); builder.add(workUnitState); } return builder.build(); }