private Optional<String> getId (String jobName, String idName) throws IOException { State state = this.stateStore.get(distributedStateStoreName, jobName, jobName); if (state == null) { return Optional.empty(); } String id = state.getProp(idName); return id == null? Optional.empty() : Optional.of(id); }
@Override public Map<String, CheckpointableWatermark> getCommittedWatermarks(Class<? extends CheckpointableWatermark> watermarkClass, Iterable<String> sourcePartitions) throws IOException { Map<String, CheckpointableWatermark> committed = new HashMap<String, CheckpointableWatermark>(); for (String sourcePartition: sourcePartitions) { CheckpointableWatermarkState watermarkState = _stateStore.get(_storeName, sourcePartition, sourcePartition); if (watermarkState != null) { CheckpointableWatermark watermark = GSON.fromJson(watermarkState.getProp(sourcePartition), watermarkClass); committed.put(sourcePartition, watermark); } } if (committed.isEmpty()) { log.warn("Didn't find any committed watermarks"); } return committed; }
@Nullable private State getOrCreate (String storeName, String jobName) throws IOException { if (this.stateStore.exists(storeName, jobName)) { return this.stateStore.get(storeName, jobName, jobName); } return new State(); }
private JobState getJobState() throws java.io.IOException { JobState jobState; // read the state from the state store if present, otherwise deserialize directly from the file if (_stateStores.haveJobStateStore()) { jobState = _stateStores.getJobStateStore().get(_jobStateFilePath.getParent().getName(), _jobStateFilePath.getName(), _jobStateFilePath.getParent().getName()); } else { jobState = new JobState(); SerializationUtils.deserializeState(_fs, _jobStateFilePath, jobState); } return jobState; }
@Test(dependsOnMethods = "testGetPreviousDatasetStatesByUrns") public void testDeleteJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertNotNull(jobState); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); zkJobStateStore.delete(TEST_JOB_NAME); jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertNull(jobState); }
@Test(dependsOnMethods = "testGetPreviousDatasetStatesByUrns") public void testDeleteJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertNotNull(jobState); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); dbJobStateStore.delete(TEST_JOB_NAME); jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertNull(jobState); }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertNotEquals(jobState.getProp("lower"), "case"); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertEquals(jobState.getProp("lower"), "case"); }
@Override public Map<String, CheckpointableWatermark> getCommittedWatermarks(Class<? extends CheckpointableWatermark> watermarkClass, Iterable<String> sourcePartitions) throws IOException { Map<String, CheckpointableWatermark> committed = new HashMap<String, CheckpointableWatermark>(); for (String sourcePartition: sourcePartitions) { CheckpointableWatermarkState watermarkState = _stateStore.get(_storeName, sourcePartition, sourcePartition); if (watermarkState != null) { CheckpointableWatermark watermark = GSON.fromJson(watermarkState.getProp(sourcePartition), watermarkClass); committed.put(sourcePartition, watermark); } } if (committed.isEmpty()) { log.warn("Didn't find any committed watermarks"); } return committed; }
private JobState getJobState() throws java.io.IOException { JobState jobState; // read the state from the state store if present, otherwise deserialize directly from the file if (_stateStores.haveJobStateStore()) { jobState = _stateStores.getJobStateStore().get(_jobStateFilePath.getParent().getName(), _jobStateFilePath.getName(), _jobStateFilePath.getParent().getName()); } else { jobState = new JobState(); SerializationUtils.deserializeState(_fs, _jobStateFilePath, jobState); } return jobState; }
@VisibleForTesting protected DistributeJobResult getResultFromUserContent() { String planningId = getPlanningJobId(this.jobPlanningProps); try { TaskState taskState = this.stateStores.getTaskStateStore().get(planningId, planningId, planningId); return new DistributeJobResult(Optional.of(taskState.getProperties()), Optional.empty()); } catch (IOException e) { return new DistributeJobResult(Optional.empty(), Optional.of(e)); } }