@Override protected void propsToJson(JsonWriter jsonWriter) throws IOException { jsonWriter.beginObject(); jsonWriter.name(ConfigurationKeys.DATASET_URN_KEY).value(getDatasetUrn()); jsonWriter.name(ConfigurationKeys.JOB_FAILURES_KEY).value(getJobFailures()); jsonWriter.endObject(); }
/** * Get a {@link Map} from dataset URNs to the latest {@link JobState.DatasetState}s. * * @param jobName the job name * @return a {@link Map} from dataset URNs to the latest {@link JobState.DatasetState}s * @throws IOException if there's something wrong reading the {@link JobState.DatasetState}s */ public Map<String, JobState.DatasetState> getLatestDatasetStatesByUrns(String jobName) throws IOException { List<JobState.DatasetState> previousDatasetStates = getAll(jobName, "%" + CURRENT_DATASET_STATE_FILE_SUFFIX + DATASET_STATE_STORE_TABLE_SUFFIX, true); Map<String, JobState.DatasetState> datasetStatesByUrns = Maps.newHashMap(); for (JobState.DatasetState previousDatasetState : previousDatasetStates) { datasetStatesByUrns.put(previousDatasetState.getDatasetUrn(), previousDatasetState); } // The dataset (job) state from the deprecated "current.jst" will be read even though // the job has transitioned to the new dataset-based mechanism if (datasetStatesByUrns.size() > 1) { datasetStatesByUrns.remove(ConfigurationKeys.DEFAULT_DATASET_URN); } return datasetStatesByUrns; }
@Override protected void writeStateSummary(JsonWriter jsonWriter) throws IOException { super.writeStateSummary(jsonWriter); jsonWriter.name("datasetUrn").value(getDatasetUrn()); } }
/** * Get a {@link Map} from dataset URNs to the latest {@link JobState.DatasetState}s. * * @param jobName the job name * @return a {@link Map} from dataset URNs to the latest {@link JobState.DatasetState}s * @throws IOException if there's something wrong reading the {@link JobState.DatasetState}s */ public Map<String, JobState.DatasetState> getLatestDatasetStatesByUrns(String jobName) throws IOException { List<JobState.DatasetState> previousDatasetStates = getAll(jobName, "%" + CURRENT_DATASET_STATE_FILE_SUFFIX + DATASET_STATE_STORE_TABLE_SUFFIX, true); Map<String, JobState.DatasetState> datasetStatesByUrns = Maps.newHashMap(); for (JobState.DatasetState previousDatasetState : previousDatasetStates) { datasetStatesByUrns.put(previousDatasetState.getDatasetUrn(), previousDatasetState); } // The dataset (job) state from the deprecated "current.jst" will be read even though // the job has transitioned to the new dataset-based mechanism if (datasetStatesByUrns.size() > 1) { datasetStatesByUrns.remove(ConfigurationKeys.DEFAULT_DATASET_URN); } return datasetStatesByUrns; }
@Override protected void writeStateSummary(JsonWriter jsonWriter) throws IOException { super.writeStateSummary(jsonWriter); jsonWriter.name("datasetUrn").value(getDatasetUrn()); } }
@Override protected void propsToJson(JsonWriter jsonWriter) throws IOException { jsonWriter.beginObject(); jsonWriter.name(ConfigurationKeys.DATASET_URN_KEY).value(getDatasetUrn()); jsonWriter.name(ConfigurationKeys.JOB_FAILURES_KEY).value(getJobFailures()); jsonWriter.endObject(); }
JobState.DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getDatasetUrn(), "Dataset" + i); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 1);
public void runTestWithMultipleDatasets(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, MultiDatasetTestSource.class.getName()); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } for (int i = 0; i < 4; i++) { List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, "Dataset" + i + "-current.jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getDatasetUrn(), "Dataset" + i); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 1); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY), "Dataset" + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
@Test(dependsOnMethods = "testGetStoreNames") public void testGetPreviousDatasetStatesByUrns() throws IOException { Map<String, JobState.DatasetState> datasetStatesByUrns = dbDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME); Assert.assertEquals(datasetStatesByUrns.size(), 3); JobState.DatasetState datasetState = datasetStatesByUrns.get(TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); datasetState = datasetStatesByUrns.get(TEST_DATASET_URN2); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN2); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 2000); datasetState = datasetStatesByUrns.get(TEST_DATASET_URN_LOWER); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN_LOWER); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 3000); }
private void testDatasetStates(List<Dataset> datasets, List<LongWatermark> watermarks, String jobName) throws IOException { Preconditions.checkArgument(datasets.size() >= 2); for (int i = 0; i < datasets.size(); i++) { JobState.DatasetState datasetState = this.fsDatasetStateStore.getLatestDatasetState(jobName, datasets.get(i).getUrn()); Assert.assertEquals(datasetState.getDatasetUrn(), datasets.get(i).getUrn()); Assert.assertEquals(datasetState.getJobName(), jobName); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 1); TaskState taskState = datasetState.getTaskStates().get(0); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); if (i < datasets.size() - 1) { Assert.assertEquals(taskState.getActualHighWatermark(LongWatermark.class).getValue(), watermarks.get(i).getValue()); } } }
/** * Get a {@link Map} from dataset URNs to the latest {@link JobState.DatasetState}s. * * @param jobName the job name * @return a {@link Map} from dataset URNs to the latest {@link JobState.DatasetState}s * @throws IOException if there's something wrong reading the {@link JobState.DatasetState}s */ public Map<String, JobState.DatasetState> getLatestDatasetStatesByUrns(String jobName) throws IOException { List<JobState.DatasetState> previousDatasetStates = getAll(jobName, new Predicate<String>() { @Override public boolean apply(String input) { return input.endsWith(CURRENT_SUFFIX); }}); Map<String, JobState.DatasetState> datasetStatesByUrns = Maps.newHashMap(); for (JobState.DatasetState previousDatasetState : previousDatasetStates) { datasetStatesByUrns.put(previousDatasetState.getDatasetUrn(), previousDatasetState); } // The dataset (job) state from the deprecated "current.jst" will be read even though // the job has transitioned to the new dataset-based mechanism if (datasetStatesByUrns.size() > 1) { datasetStatesByUrns.remove(ConfigurationKeys.DEFAULT_DATASET_URN); } return datasetStatesByUrns; }
@Test(dependsOnMethods = "testGetStoreNames") public void testGetPreviousDatasetStatesByUrns() throws IOException { Map<String, JobState.DatasetState> datasetStatesByUrns = zkDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME); Assert.assertEquals(datasetStatesByUrns.size(), 2); JobState.DatasetState datasetState = datasetStatesByUrns.get(TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); datasetState = datasetStatesByUrns.get(TEST_DATASET_URN2); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN2); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 2000); }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = this.fsDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = dbDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = zkDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testGetDatasetState") public void testGetPreviousDatasetStatesByUrns() throws IOException { Map<String, JobState.DatasetState> datasetStatesByUrns = this.fsDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME); Assert.assertEquals(datasetStatesByUrns.size(), 1); JobState.DatasetState datasetState = datasetStatesByUrns.get(TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); }