/** * Create a {@link Map} from dataset URNs (as being specified by {@link ConfigurationKeys#DATASET_URN_KEY} to * {@link DatasetState} objects that represent the dataset states and store {@link TaskState}s corresponding * to the datasets. * * <p> * {@link TaskState}s that do not have {@link ConfigurationKeys#DATASET_URN_KEY} set will be added to * the dataset state belonging to {@link ConfigurationKeys#DEFAULT_DATASET_URN}. * </p> * * @return a {@link Map} from dataset URNs to {@link DatasetState}s representing the dataset states */ public Map<String, DatasetState> createDatasetStatesByUrns() { Map<String, DatasetState> datasetStatesByUrns = Maps.newHashMap(); for (TaskState taskState : this.taskStates.values()) { String datasetUrn = createDatasetUrn(datasetStatesByUrns, taskState); datasetStatesByUrns.get(datasetUrn).incrementTaskCount(); datasetStatesByUrns.get(datasetUrn).addTaskState(taskState); } for (TaskState taskState : this.skippedTaskStates.values()) { String datasetUrn = createDatasetUrn(datasetStatesByUrns, taskState); datasetStatesByUrns.get(datasetUrn).addSkippedTaskState(taskState); } return ImmutableMap.copyOf(datasetStatesByUrns); }
taskState.setTaskId("task123"); taskState.setProp("key", "value"); dataset2State.addTaskState(taskState);
public void persistDatasetState(List<Dataset> datasets, List<LongWatermark> watermarks, String jobName) throws IOException { Preconditions.checkArgument(datasets.size() >= 2); for (int i = 0; i < datasets.size(); i++) { String datasetUrn = datasets.get(i).getUrn(); JobState.DatasetState datasetState = new JobState.DatasetState(jobName, TEST_JOB_ID); datasetState.setDatasetUrn(datasetUrn); datasetState.setState(JobState.RunningState.COMMITTED); datasetState.setId(datasetUrn); datasetState.setStartTime(this.startTime); datasetState.setEndTime(this.startTime + 1000); datasetState.setDuration(1000); TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); if (i < datasets.size() - 1) { taskState.setActualHighWatermark(watermarks.get(i)); } datasetState.addTaskState(taskState); this.fsDatasetStateStore.persistDatasetState(datasetUrn, datasetState); } }
/** * Create a {@link Map} from dataset URNs (as being specified by {@link ConfigurationKeys#DATASET_URN_KEY} to * {@link DatasetState} objects that represent the dataset states and store {@link TaskState}s corresponding * to the datasets. * * <p> * {@link TaskState}s that do not have {@link ConfigurationKeys#DATASET_URN_KEY} set will be added to * the dataset state belonging to {@link ConfigurationKeys#DEFAULT_DATASET_URN}. * </p> * * @return a {@link Map} from dataset URNs to {@link DatasetState}s representing the dataset states */ public Map<String, DatasetState> createDatasetStatesByUrns() { Map<String, DatasetState> datasetStatesByUrns = Maps.newHashMap(); for (TaskState taskState : this.taskStates.values()) { String datasetUrn = createDatasetUrn(datasetStatesByUrns, taskState); datasetStatesByUrns.get(datasetUrn).incrementTaskCount(); datasetStatesByUrns.get(datasetUrn).addTaskState(taskState); } for (TaskState taskState : this.skippedTaskStates.values()) { String datasetUrn = createDatasetUrn(datasetStatesByUrns, taskState); datasetStatesByUrns.get(datasetUrn).addSkippedTaskState(taskState); } return ImmutableMap.copyOf(datasetStatesByUrns); }
taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); datasetState.addTaskState(taskState);
taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); datasetState.addTaskState(taskState);
@Test(dependsOnMethods = "testGetJobState") public void testPersistDatasetState() throws IOException { JobState.DatasetState datasetState = new JobState.DatasetState(TEST_JOB_NAME, TEST_JOB_ID); datasetState.setDatasetUrn(TEST_DATASET_URN); datasetState.setState(JobState.RunningState.COMMITTED); datasetState.setId(TEST_DATASET_URN); datasetState.setStartTime(this.startTime); datasetState.setEndTime(this.startTime + 1000); datasetState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); datasetState.addTaskState(taskState); } zkDatasetStateStore.persistDatasetState(TEST_DATASET_URN, datasetState); // persist a second dataset state to test that retrieval of multiple dataset states works datasetState.setDatasetUrn(TEST_DATASET_URN2); datasetState.setId(TEST_DATASET_URN2); datasetState.setDuration(2000); zkDatasetStateStore.persistDatasetState(TEST_DATASET_URN2, datasetState); // second job name for testing getting store names in a later test case datasetState.setJobName(TEST_JOB_NAME2); zkDatasetStateStore.persistDatasetState(TEST_DATASET_URN2, datasetState); }
@Test(dependsOnMethods = "testGetJobState") public void testPersistDatasetState() throws IOException { JobState.DatasetState datasetState = new JobState.DatasetState(TEST_JOB_NAME, TEST_JOB_ID); datasetState.setDatasetUrn(TEST_DATASET_URN); datasetState.setState(JobState.RunningState.COMMITTED); datasetState.setId(TEST_DATASET_URN); datasetState.setStartTime(this.startTime); datasetState.setEndTime(this.startTime + 1000); datasetState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); datasetState.addTaskState(taskState); } this.fsDatasetStateStore.persistDatasetState(TEST_DATASET_URN, datasetState); }