@Override public CombinedWorkUnitAndDatasetState getCombinedWorkUnitAndDatasetState(String datasetUrn) throws Exception { Map<String, JobState.DatasetState> datasetStateMap = ImmutableMap.of(); List<WorkUnitState> workUnitStates = new ArrayList<>(); if (Strings.isNullOrEmpty(datasetUrn)) { datasetStateMap = this.datasetStateStore.getLatestDatasetStatesByUrns(this.jobName); workUnitStates = JobState.workUnitStatesFromDatasetStates(datasetStateMap.values()); } else { JobState.DatasetState datasetState = (JobState.DatasetState) this.datasetStateStore.getLatestDatasetState(this.jobName, datasetUrn); if (datasetState != null) { datasetStateMap = ImmutableMap.of(datasetUrn, datasetState); workUnitStates = JobState.workUnitStatesFromDatasetStates(Arrays.asList(datasetState)); } } return new CombinedWorkUnitAndDatasetState(workUnitStates, datasetStateMap); } }
@Override public boolean isCompleted() throws IOException { Preconditions.checkNotNull(this.datasetState); return this.datasetState .equals(getDatasetStateStore().getLatestDatasetState(this.datasetState.getJobName(), this.datasetUrn)); }
@Override public CombinedWorkUnitAndDatasetState getCombinedWorkUnitAndDatasetState(String datasetUrn) throws Exception { Map<String, JobState.DatasetState> datasetStateMap = ImmutableMap.of(); List<WorkUnitState> workUnitStates = new ArrayList<>(); if (Strings.isNullOrEmpty(datasetUrn)) { datasetStateMap = this.datasetStateStore.getLatestDatasetStatesByUrns(this.jobName); workUnitStates = JobState.workUnitStatesFromDatasetStates(datasetStateMap.values()); } else { JobState.DatasetState datasetState = (JobState.DatasetState) this.datasetStateStore.getLatestDatasetState(this.jobName, datasetUrn); if (datasetState != null) { datasetStateMap = ImmutableMap.of(datasetUrn, datasetState); workUnitStates = JobState.workUnitStatesFromDatasetStates(Arrays.asList(datasetState)); } } return new CombinedWorkUnitAndDatasetState(workUnitStates, datasetStateMap); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = zkDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = dbDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Override protected WorkUnit workUnitForDataset(Dataset dataset) { WorkUnit workUnit = new WorkUnit(); if(isDatasetStateStoreEnabled) { JobState.DatasetState datasetState = null; try { datasetState = (JobState.DatasetState) this.fsDatasetStateStore.getLatestDatasetState(this.jobName, dataset.getUrn()); } catch (IOException e) { throw new RuntimeException(e); } LongWatermark previousWatermark; if(datasetState != null) { previousWatermark = datasetState.getTaskStatesAsWorkUnitStates().get(0).getActualHighWatermark(LongWatermark.class); } else { previousWatermark = new LongWatermark(0); } workUnit.setWatermarkInterval(new WatermarkInterval(previousWatermark, new LongWatermark(LAST_PROCESSED_TS))); } return workUnit; }
@Override protected WorkUnit workUnitForDatasetPartition(PartitionableDataset.DatasetPartition partition) { WorkUnit workUnit = new WorkUnit(); if(isDatasetStateStoreEnabled) { String datasetUrn = partition.getDataset().getUrn()+"@"+partition.getUrn(); JobState.DatasetState datasetState = null; try { datasetState = (JobState.DatasetState) this.fsDatasetStateStore.getLatestDatasetState(this.jobName, datasetUrn); } catch (IOException e) { throw new RuntimeException(e); } LongWatermark previousWatermark; if(datasetState != null) { previousWatermark = datasetState.getTaskStatesAsWorkUnitStates().get(0).getActualHighWatermark(LongWatermark.class); } else { previousWatermark = new LongWatermark(0); } workUnit.setWatermarkInterval(new WatermarkInterval(previousWatermark, new LongWatermark(LAST_PROCESSED_TS))); } return workUnit; }
@Override public boolean isCompleted() throws IOException { Preconditions.checkNotNull(this.datasetState); return this.datasetState .equals(getDatasetStateStore().getLatestDatasetState(this.datasetState.getJobName(), this.datasetUrn)); }