@Override public Void call() throws Exception { if (dataset instanceof CleanableDataset) { ((CleanableDataset) dataset).clean(); } return null; } });
protected void cleanImpl(Collection<T> deletableVersions) throws IOException { this.fsCleanableHelper.clean(deletableVersions, this); }
@Override public Dataset datasetAtPath(Path path) throws IOException { return new TrackingDataset(this.fs, this.props, path); } }
@Override public List<VersionFinderAndPolicy<T>> getVersionFindersAndPolicies() { return ImmutableList .<VersionFinderAndPolicy<T>> of(new VersionFinderAndPolicy<>(getRetentionPolicy(), getVersionFinder())); } }
/** * Constructor for {@link MultiVersionCleanableDatasetBase}. * @param fs {@link org.apache.hadoop.fs.FileSystem} where files are located. * @param properties {@link java.util.Properties} for object. * @param simulate whether to simulate deletes. * @param skipTrash if true, delete files and directories immediately. * @param deleteEmptyDirectories if true, newly empty parent directories will be deleted. * @param deleteAsOwner if true, all deletions will be executed as the owner of the file / directory. * @param log logger to use. * @param isDatasetBlacklisted if true, clean will be skipped for this dataset * * @throws IOException */ public MultiVersionCleanableDatasetBase(FileSystem fs, Properties properties, boolean simulate, boolean skipTrash, boolean deleteEmptyDirectories, boolean deleteAsOwner, boolean isDatasetBlacklisted, Logger log) throws IOException { this.log = log; this.fsCleanableHelper = new FsCleanableHelper(fs, properties, simulate, skipTrash, deleteEmptyDirectories, deleteAsOwner, log); this.fs = fs; this.simulate = simulate; this.skipTrash = skipTrash; this.deleteEmptyDirectories = deleteEmptyDirectories; this.trash = this.fsCleanableHelper.getTrash(); this.deleteAsOwner = deleteAsOwner; this.isDatasetBlacklisted = isDatasetBlacklisted; }
private void initWithRetentionPolicy(Config config, Properties jobProps, String retentionPolicyKey, String versionFinderKey) { this.versionFindersAndPolicies .add(new VersionFinderAndPolicy<>(createRetentionPolicy(config.getString(retentionPolicyKey), config, jobProps), createVersionFinder(config.getString(versionFinderKey), config, jobProps))); }
@Override public ConfigurableCleanableDataset<FileSystemDatasetVersion> datasetAtPath(Path path) throws IOException { return new ConfigurableCleanableDataset<>(this.fs, this.props, path, LoggerFactory.getLogger(ConfigurableCleanableDataset.class)); } }
/** * Delete all {@link FileSystemDatasetVersion}s <code>deletableVersions</code> and also delete any empty parent directories. * * @param fsDataset to which the version belongs. */ public void clean(final Collection<? extends FileSystemDatasetVersion> deletableVersions, final FileSystemDataset fsDataset) throws IOException { if (deletableVersions.isEmpty()) { log.warn("No deletable dataset version can be found. Ignoring."); return; } Set<Path> possiblyEmptyDirectories = new HashSet<>(); for (FileSystemDatasetVersion fsdv : deletableVersions) { clean(fsdv, possiblyEmptyDirectories); } cleanEmptyDirectories(possiblyEmptyDirectories, fsDataset); }
@Override public String datasetURN() { return this.datasetRoot().toString(); }
@Override public Dataset datasetAtPath(Path path) throws IOException { return new ModificationTimeDataset(this.fs, this.props, path); } }
@Override public Dataset datasetAtPath(Path path) throws IOException { return new SnapshotDataset(this.fs, this.props, path); }
@SuppressWarnings("unchecked") public VersionFinderAndPolicy<T> build() { VersionSelectionPolicy<T> localVersionSelectionPolicy; List<RetentionAction> localRetentionActions; if (this.versionSelectionPolicy == null) { localVersionSelectionPolicy = (VersionSelectionPolicy<T>) new SelectNothingPolicy(new Properties()); } else { localVersionSelectionPolicy = this.versionSelectionPolicy; } if (this.retentionActions == null) { localRetentionActions = Lists.newArrayList(); } else { localRetentionActions = Lists.newArrayList(this.retentionActions); } return new VersionFinderAndPolicy<T>(localVersionSelectionPolicy, this.versionFinder, localRetentionActions); } }
protected CleanableHiveDataset createHiveDataset(Table table, Config datasetConfig) throws IOException { return new CleanableHiveDataset(super.fs, super.clientPool, new org.apache.hadoop.hive.ql.metadata.Table(table), super.properties, datasetConfig); }
@Override public List<DatasetStoreDataset> findDatasets() throws IOException { return super.findDatasets().stream() .map(dataset -> new TimeBasedDatasetStoreDataset(dataset.getKey(), dataset.getDatasetStateStoreMetadataEntries(), props)) .collect(Collectors.toList()); } }
public TimeBasedDatasetStoreDataset(Key key, List<DatasetStateStoreEntryManager> entries, Properties props) { super(key, entries); this.versionFinder = new TimestampedDatasetStateStoreVersionFinder(); Config propsAsConfig = ConfigUtils.propertiesToConfig(props); // strip the retention config namespace since the selection policy looks for configuration without the namespace Config retentionConfig = ConfigUtils.getConfigOrEmpty(propsAsConfig, ConfigurableCleanableDataset.RETENTION_CONFIGURATION_KEY); Config retentionConfigWithFallback = retentionConfig.withFallback(propsAsConfig); this.versionSelectionPolicy = createSelectionPolicy(ConfigUtils.getString(retentionConfigWithFallback, SELECTION_POLICY_CLASS_KEY, DEFAULT_SELECTION_POLICY_CLASS), retentionConfigWithFallback, props); }
@Override public Dataset datasetAtPath(Path path) throws IOException { return new ConfigurableCleanableDataset<DatasetVersion>(this.getFsForDataset(path), this.props, path); }
@Override public Void call() throws Exception { if (dataset instanceof CleanableDataset) { ((CleanableDataset) dataset).clean(); } else { log.warn( "Not an instance of " + CleanableDataset.class + " Dataset won't be cleaned " + dataset.datasetURN()); } return null; } });
@Override public String toString() { return datasetRoot().toString(); }
@Override public Void call() throws Exception { // Process each {@link Config}, find dataset and add those into the datasets Config c = confClient.getConfig(u); Dataset datasetForConfig = new ConfigurableCleanableDataset(fileSystem, p, new Path(c.getString(DATASET_PATH)), c, log); datasets.add(datasetForConfig); return null; } };
@Override public ConfigurableCleanableDataset<FileSystemDatasetVersion> datasetAtPath(Path path) throws IOException { try { return new ConfigurableCleanableDataset<>(this.fs, this.props, path, this.client .getConfig(this.props.getProperty(ConfigurationKeys.CONFIG_MANAGEMENT_STORE_URI) + path.toString()), LoggerFactory.getLogger(ConfigurableCleanableDataset.class)); } catch (VersionDoesNotExistException | ConfigStoreFactoryDoesNotExistsException | ConfigStoreCreationException | URISyntaxException e) { throw new IllegalArgumentException(e); } } }