@Override public List<DatasetStoreDataset> findDatasets() throws IOException { return super.findDatasets().stream() .map(dataset -> new TimeBasedDatasetStoreDataset(dataset.getKey(), dataset.getDatasetStateStoreMetadataEntries(), props)) .collect(Collectors.toList()); } }
@Override public List<DatasetStoreDataset> findDatasets() throws IOException { List<DatasetStateStoreEntryManager> entries = this.store.getMetadataForTables(this.predicate); Map<DatasetStoreDataset.Key, List<DatasetStateStoreEntryManager>> entriesGroupedByDataset = entries.stream().collect(Collectors.groupingBy(DatasetStoreDataset.Key::new)); return entriesGroupedByDataset.entrySet().stream(). map(entry -> new DatasetStoreDataset(entry.getKey(), entry.getValue())).collect(Collectors.toList()); }
@Test public void testCleanStateStore() throws IOException { File tmpDir = Files.createTempDir(); tmpDir.deleteOnExit(); FileSystem fs = FileSystem.getLocal(new Configuration()); FsDatasetStateStore store = new FsDatasetStateStore(fs, tmpDir.getAbsolutePath()); store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id1")); store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id2")); store.persistDatasetState("dataset1", new JobState.DatasetState("job2", "job2_id1")); store.persistDatasetState("", new JobState.DatasetState("job3", "job3_id1")); Properties props = new Properties(); props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, tmpDir.getAbsolutePath()); props.setProperty("selection.timeBased.lookbackTime", "0m"); TimeBasedDatasetStoreDatasetFinder datasetFinder = new TimeBasedDatasetStoreDatasetFinder(fs, props); List<DatasetStoreDataset> datasets = datasetFinder.findDatasets(); for (DatasetStoreDataset dataset : datasets) { ((CleanableDataset) dataset).clean(); File jobDir = new File(tmpDir.getAbsolutePath(), dataset.getKey().getStoreName()); Assert.assertEquals(jobDir.list().length, 1); } } }
@Override public Collection<TimestampedDatasetStateStoreVersion> findDatasetVersions(Dataset dataset) throws IOException { DatasetStoreDataset storeDataset = ((DatasetStoreDataset) dataset); List<TimestampedDatasetStateStoreVersion> versions = Lists.newArrayList(); for (DatasetStateStoreEntryManager entry : storeDataset.getDatasetStateStoreMetadataEntries()) { versions.add(new TimestampedDatasetStateStoreVersion(entry)); } return versions; } }
@Override public Collection<TimestampedDatasetStateStoreVersion> findDatasetVersions(Dataset dataset) throws IOException { DatasetStoreDataset storeDataset = ((DatasetStoreDataset) dataset); List<TimestampedDatasetStateStoreVersion> versions = Lists.newArrayList(); for (DatasetStateStoreEntryManager entry : storeDataset.getDatasetStateStoreMetadataEntries()) { versions.add(new TimestampedDatasetStateStoreVersion(entry)); } return versions; } }
@Override public List<DatasetStoreDataset> findDatasets() throws IOException { return super.findDatasets().stream() .map(dataset -> new TimeBasedDatasetStoreDataset(dataset.getKey(), dataset.getDatasetStateStoreMetadataEntries(), props)) .collect(Collectors.toList()); } }
@Override public List<DatasetStoreDataset> findDatasets() throws IOException { List<DatasetStateStoreEntryManager> entries = this.store.getMetadataForTables(this.predicate); Map<DatasetStoreDataset.Key, List<DatasetStateStoreEntryManager>> entriesGroupedByDataset = entries.stream().collect(Collectors.groupingBy(DatasetStoreDataset.Key::new)); return entriesGroupedByDataset.entrySet().stream(). map(entry -> new DatasetStoreDataset(entry.getKey(), entry.getValue())).collect(Collectors.toList()); }