Iterable<CheckpointableWatermarkState> getAllCommittedWatermarks() throws IOException { return _stateStore.getAll(_storeName); }
public List<State> getAllStates() throws IOException { return this.stateStore.getAll(distributedStateStoreName); }
@Override public Void call() throws Exception { TaskState taskState = taskStateStore.getAll(outputTaskStateDir.getName(), taskStateName).get(0); taskStateQueue.add(taskState); taskStateStore.delete(outputTaskStateDir.getName(), taskStateName); return null; } }, "Deserialize state for " + taskStateName);
private List<WorkUnit> getWorkUnits() throws IOException { String fileName = _workUnitFilePath.getName(); String storeName = _workUnitFilePath.getParent().getName(); WorkUnit workUnit; if (_workUnitFilePath.getName().endsWith(AbstractJobLauncher.MULTI_WORK_UNIT_FILE_EXTENSION)) { workUnit = _stateStores.getMwuStateStore().getAll(storeName, fileName).get(0); } else { workUnit = _stateStores.getWuStateStore().getAll(storeName, fileName).get(0); } // The list of individual WorkUnits (flattened) to run List<WorkUnit> workUnits = Lists.newArrayList(); if (workUnit instanceof MultiWorkUnit) { // Flatten the MultiWorkUnit so the job configuration properties can be added to each individual WorkUnits List<WorkUnit> flattenedWorkUnits = JobLauncherUtils.flattenWorkUnits(((MultiWorkUnit) workUnit).getWorkUnits()); workUnits.addAll(flattenedWorkUnits); } else { workUnits.add(workUnit); } return workUnits; }
/** * Convert a single {@link JobState} of the given job instance. * * @param jobName job name * @param jobId job ID * @param writer {@link java.io.Writer} to write the json document * @throws IOException */ public void convert(String jobName, String jobId, Writer writer) throws IOException { List<? extends JobState> jobStates = this.jobStateStore.getAll(jobName, jobId + JOB_STATE_STORE_TABLE_SUFFIX); if (jobStates.isEmpty()) { LOGGER.warn(String.format("No job state found for job with name %s and id %s", jobName, jobId)); return; } try (JsonWriter jsonWriter = new JsonWriter(writer)) { jsonWriter.setIndent("\t"); // There should be only a single job state writeJobState(jsonWriter, jobStates.get(0)); } }
/** * Convert all past {@link JobState}s of the given job. * * @param jobName job name * @param writer {@link java.io.Writer} to write the json document * @throws IOException */ public void convertAll(String jobName, Writer writer) throws IOException { List<? extends JobState> jobStates = this.jobStateStore.getAll(jobName); if (jobStates.isEmpty()) { LOGGER.warn(String.format("No job state found for job with name %s", jobName)); return; } try (JsonWriter jsonWriter = new JsonWriter(writer)) { jsonWriter.setIndent("\t"); writeJobStates(jsonWriter, jobStates); } }
@Test(dependsOnMethods = { "testPut" }) public void testGet() throws IOException { List<State> states = this.stateStore.getAll("testStore", "testTable"); Assert.assertEquals(states.size(), 3); Assert.assertEquals(states.get(0).getProp("k1"), "v1"); Assert.assertEquals(states.get(1).getProp("k2"), "v2"); Assert.assertEquals(states.get(2).getProp("k3"), "v3"); }
@Test(dependsOnMethods = { "testCreateAlias" }) public void testGetAlias() throws IOException { List<State> states = this.stateStore.getAll("testStore", "testTable1"); Assert.assertEquals(states.size(), 3); Assert.assertEquals(states.get(0).getProp("k1"), "v1"); Assert.assertEquals(states.get(1).getProp("k2"), "v2"); Assert.assertEquals(states.get(2).getProp("k3"), "v3"); }
@Test public void testBackwardsCompat() throws IOException { // Tests with a state store that was saved before the WritableShim changes Config bwConfig = ConfigFactory.load(config); URL path = getClass().getResource("/backwardsCompatTestStore"); Assert.assertNotNull(path); bwConfig = bwConfig.withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(path.toString())); StateStore<State> bwStateStore = stateStoreFactory.createStateStore(bwConfig, State.class); Assert.assertTrue(bwStateStore.exists("testStore", "testTable")); List<State> states = bwStateStore.getAll("testStore", "testTable"); Assert.assertEquals(states.size(), 3); Assert.assertEquals(states.get(0).getProp("k1"), "v1"); Assert.assertEquals(states.get(1).getProp("k2"), "v2"); Assert.assertEquals(states.get(2).getProp("k3"), "v3"); }
public void runTestWithCancellation(final Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); Closer closer = Closer.create(); try { final JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); final AtomicBoolean isCancelled = new AtomicBoolean(false); // This thread will cancel the job after some time Thread thread = new Thread(new Runnable() { @Override public void run() { try { Thread.sleep(500); jobLauncher.cancelJob(null); isCancelled.set(true); } catch (Exception je) { // Ignored } } }); thread.start(); jobLauncher.launchJob(null); Assert.assertTrue(isCancelled.get()); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); Assert.assertTrue(datasetStateList.isEmpty()); }
List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, "Dataset0-current.jst"); JobState.DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); } else { Assert.assertTrue(this.datasetStateStore.getAll(jobName, "Dataset0-current.jst").isEmpty()); this.datasetStateStore.getAll(jobName, "Dataset" + i + "-current.jst"); JobState.DatasetState datasetState = datasetStateList.get(0);
private void verifyJobState(int run) throws IOException { List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(JOB_NAME, "current.jst"); Assert.assertEquals(datasetStateList.size(), 1); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getTaskStates().size(), DummySource.NUM_WORK_UNITS); for (TaskState taskState : jobState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getProp(FOO), BAR); // Check if the low watermark is properly kept track of int expectedLowWatermark = (run - 1) * DummySource.NUM_WORK_UNITS * DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR + taskState.getPropAsInt(WORK_UNIT_INDEX_KEY) * DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR + 1; Assert.assertEquals(taskState.getPropAsInt(ConfigurationKeys.WORK_UNIT_LOW_WATER_MARK_KEY), expectedLowWatermark); // Check if the high watermark is properly kept track of int expectedHighWatermark = expectedLowWatermark + DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR - 1; Assert.assertEquals(taskState.getPropAsInt(ConfigurationKeys.WORK_UNIT_HIGH_WATER_MARK_KEY), expectedHighWatermark); Assert.assertEquals(taskState.getPropAsInt(LAST_READ_RECORD_KEY), expectedHighWatermark); } }
public void runTestWithPullLimit(Properties jobProps, long limit) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXTRACTED), limit); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_ROWS_WRITTEN), limit); } }
public void runTestWithFork(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); try (JobLauncher jobLauncher = JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)) { jobLauncher.launchJob(null); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); FileSystem lfs = FileSystem.getLocal(new Configuration()); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Path path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR), new Path(taskState.getExtract().getOutputFilePath(), "fork_0")); Assert.assertTrue(lfs.exists(path)); Assert.assertEquals(lfs.listStatus(path).length, 2); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".0"), TestExtractor.TOTAL_RECORDS); path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR), new Path(taskState.getExtract().getOutputFilePath(), "fork_1")); Assert.assertTrue(lfs.exists(path)); Assert.assertEquals(lfs.listStatus(path).length, 2); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".1"), TestExtractor.TOTAL_RECORDS); } }
public void runTestWithMultipleDatasets(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, MultiDatasetTestSource.class.getName()); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } for (int i = 0; i < 4; i++) { List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, "Dataset" + i + "-current.jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getDatasetUrn(), "Dataset" + i); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 1); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY), "Dataset" + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
public void runTestWithCommitSuccessfulTasksPolicy(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString()); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful"); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, TestSourceWithFaultyExtractor.class.getName()); jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0"); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getCompletedTasks(), 4); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith("0")) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.FAILED); } else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0);
public void runTest(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); JobContext jobContext = null; Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); jobContext = ((AbstractJobLauncher) jobLauncher).getJobContext(); } finally { closer.close(); } Assert.assertTrue(jobContext.getJobMetricsOptional().isPresent()); String jobMetricContextTags = jobContext.getJobMetricsOptional().get().getMetricContext().getTags().toString(); Assert.assertTrue(jobMetricContextTags.contains(ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME), ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME + " tag missing in job metric context tags."); List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } }
Iterable<CheckpointableWatermarkState> getAllCommittedWatermarks() throws IOException { return _stateStore.getAll(_storeName); }
@Override public Void call() throws Exception { TaskState taskState = taskStateStore.getAll(outputTaskStateDir.getName(), taskStateName).get(0); taskStateQueue.add(taskState); taskStateStore.delete(outputTaskStateDir.getName(), taskStateName); return null; } }, "Deserialize state for " + taskStateName);