private static void migrateStateForJob(DatasetStateStore srcDatasetStateStore, DatasetStateStore dstDatasetStateStore, String jobName, boolean deleteFromSource) throws IOException { Map<String, JobState.DatasetState> map = srcDatasetStateStore.getLatestDatasetStatesByUrns(jobName); for (Map.Entry<String, JobState.DatasetState> entry : map.entrySet()) { dstDatasetStateStore.persistDatasetState(entry.getKey(), entry.getValue()); } if (deleteFromSource) { try { srcDatasetStateStore.delete(jobName); } catch (IOException ioe) { log.warn("The source state store has been deleted", ioe); } } }
@Override public CombinedWorkUnitAndDatasetState getCombinedWorkUnitAndDatasetState(String datasetUrn) throws Exception { Map<String, JobState.DatasetState> datasetStateMap = ImmutableMap.of(); List<WorkUnitState> workUnitStates = new ArrayList<>(); if (Strings.isNullOrEmpty(datasetUrn)) { datasetStateMap = this.datasetStateStore.getLatestDatasetStatesByUrns(this.jobName); workUnitStates = JobState.workUnitStatesFromDatasetStates(datasetStateMap.values()); } else { JobState.DatasetState datasetState = (JobState.DatasetState) this.datasetStateStore.getLatestDatasetState(this.jobName, datasetUrn); if (datasetState != null) { datasetStateMap = ImmutableMap.of(datasetUrn, datasetState); workUnitStates = JobState.workUnitStatesFromDatasetStates(Arrays.asList(datasetState)); } } return new CombinedWorkUnitAndDatasetState(workUnitStates, datasetStateMap); } }
@Test(dependsOnMethods = "testGetStoreNames") public void testGetPreviousDatasetStatesByUrns() throws IOException { Map<String, JobState.DatasetState> datasetStatesByUrns = zkDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME); Assert.assertEquals(datasetStatesByUrns.size(), 2); JobState.DatasetState datasetState = datasetStatesByUrns.get(TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); datasetState = datasetStatesByUrns.get(TEST_DATASET_URN2); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN2); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 2000); }
@Test(dependsOnMethods = "testGetStoreNames") public void testGetPreviousDatasetStatesByUrns() throws IOException { Map<String, JobState.DatasetState> datasetStatesByUrns = dbDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME); Assert.assertEquals(datasetStatesByUrns.size(), 3); JobState.DatasetState datasetState = datasetStatesByUrns.get(TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); datasetState = datasetStatesByUrns.get(TEST_DATASET_URN2); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN2); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 2000); datasetState = datasetStatesByUrns.get(TEST_DATASET_URN_LOWER); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN_LOWER); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 3000); }
private static void migrateStateForJob(DatasetStateStore srcDatasetStateStore, DatasetStateStore dstDatasetStateStore, String jobName, boolean deleteFromSource) throws IOException { Map<String, JobState.DatasetState> map = srcDatasetStateStore.getLatestDatasetStatesByUrns(jobName); for (Map.Entry<String, JobState.DatasetState> entry : map.entrySet()) { dstDatasetStateStore.persistDatasetState(entry.getKey(), entry.getValue()); } if (deleteFromSource) { try { srcDatasetStateStore.delete(jobName); } catch (IOException ioe) { log.warn("The source state store has been deleted", ioe); } } }
@Override public CombinedWorkUnitAndDatasetState getCombinedWorkUnitAndDatasetState(String datasetUrn) throws Exception { Map<String, JobState.DatasetState> datasetStateMap = ImmutableMap.of(); List<WorkUnitState> workUnitStates = new ArrayList<>(); if (Strings.isNullOrEmpty(datasetUrn)) { datasetStateMap = this.datasetStateStore.getLatestDatasetStatesByUrns(this.jobName); workUnitStates = JobState.workUnitStatesFromDatasetStates(datasetStateMap.values()); } else { JobState.DatasetState datasetState = (JobState.DatasetState) this.datasetStateStore.getLatestDatasetState(this.jobName, datasetUrn); if (datasetState != null) { datasetStateMap = ImmutableMap.of(datasetUrn, datasetState); workUnitStates = JobState.workUnitStatesFromDatasetStates(Arrays.asList(datasetState)); } } return new CombinedWorkUnitAndDatasetState(workUnitStates, datasetStateMap); } }