@Override public Void call() throws Exception { if (dataset instanceof CleanableDataset) { ((CleanableDataset) dataset).clean(); } return null; } });
@Override public Void call() throws Exception { if (dataset instanceof CleanableDataset) { ((CleanableDataset) dataset).clean(); } else { log.warn( "Not an instance of " + CleanableDataset.class + " Dataset won't be cleaned " + dataset.datasetURN()); } return null; } });
@Test public void testCleanStateStore() throws IOException { File tmpDir = Files.createTempDir(); tmpDir.deleteOnExit(); FileSystem fs = FileSystem.getLocal(new Configuration()); FsDatasetStateStore store = new FsDatasetStateStore(fs, tmpDir.getAbsolutePath()); store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id1")); store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id2")); store.persistDatasetState("dataset1", new JobState.DatasetState("job2", "job2_id1")); store.persistDatasetState("", new JobState.DatasetState("job3", "job3_id1")); Properties props = new Properties(); props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, tmpDir.getAbsolutePath()); props.setProperty("selection.timeBased.lookbackTime", "0m"); TimeBasedDatasetStoreDatasetFinder datasetFinder = new TimeBasedDatasetStoreDatasetFinder(fs, props); List<DatasetStoreDataset> datasets = datasetFinder.findDatasets(); for (DatasetStoreDataset dataset : datasets) { ((CleanableDataset) dataset).clean(); File jobDir = new File(tmpDir.getAbsolutePath(), dataset.getKey().getStoreName()); Assert.assertEquals(jobDir.list().length, 1); } } }
((CleanableDataset) dataset).clean();
((CleanableDataset) dataset).clean();
@Override public Void call() throws Exception { if (dataset instanceof CleanableDataset) { ((CleanableDataset) dataset).clean(); } return null; } });