@Nullable @Override public Set<DatasetVersion> apply(VersionSelectionPolicy<DatasetVersion> input) { return Sets.newHashSet(input.listSelectedVersions(allVersions)); } }));
/** * Applies {@link #selectionPolicy} on <code>allVersions</code> and modifies permission/owner to the selected {@link DatasetVersion}s * where necessary. * <p> * This action only available for {@link FileSystemDatasetVersion}. It simply skips the operation if a different type * of {@link DatasetVersion} is passed. * </p> * {@inheritDoc} * @see org.apache.gobblin.data.management.retention.action.RetentionAction#execute(java.util.List) */ @Override public void execute(List<DatasetVersion> allVersions) throws IOException { // Select version on which access control actions need to performed for (DatasetVersion datasetVersion : this.selectionPolicy.listSelectedVersions(allVersions)) { executeOnVersion(datasetVersion); } }
@Override public Collection<CopyableFile> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { log.info(String.format("Getting copyable files at root path: %s", this.datasetRoot)); List<TimestampedDatasetVersion> versions = Lists.newArrayList(this.datasetVersionFinder.findDatasetVersions(this)); if (versions.isEmpty()) { log.warn("No dataset version can be found. Ignoring."); return Lists.newArrayList(); } Collection<TimestampedDatasetVersion> copyableVersions = this.versionSelectionPolicy.listSelectedVersions(versions); ConcurrentLinkedQueue<CopyableFile> copyableFileList = new ConcurrentLinkedQueue<>(); List<Future<?>> futures = Lists.newArrayList(); for (TimestampedDatasetVersion copyableVersion : copyableVersions) { futures.add(this.executor.submit(this.getCopyableFileGenetator(targetFs, configuration, copyableVersion, copyableFileList))); } try { for (Future<?> future : futures) { future.get(); } } catch (ExecutionException | InterruptedException e) { throw new IOException("Failed to generate copyable files.", e); } finally { ExecutorsUtils.shutdownExecutorService(executor, Optional.of(log)); } return copyableFileList; }
@Override public void clean() throws IOException { List<T> versions = Lists.newArrayList(this.getVersionFinder().findDatasetVersions(this)); Collections.sort(versions, Collections.reverseOrder()); Collection<T> deletableVersions = this.getVersionSelectionPolicy().listSelectedVersions(versions); for (Object version : deletableVersions) { ((DatasetStateStoreVersion) version).getEntry().delete(); } } }
Collection<HiveDatasetVersion> deletableVersions = this.hiveSelectionPolicy.listSelectedVersions(versions);
Collection<FileSystemDatasetVersion> versionsSelected = selector.listSelectedVersions(versions);
Collection<T> deletableVersions = selectionPolicy.listSelectedVersions(versions);
@Nullable @Override public Set<DatasetVersion> apply(VersionSelectionPolicy<DatasetVersion> input) { return Sets.newHashSet(input.listSelectedVersions(allVersions)); } }));
/** * Applies {@link #selectionPolicy} on <code>allVersions</code> and modifies permission/owner to the selected {@link DatasetVersion}s * where necessary. * <p> * This action only available for {@link FileSystemDatasetVersion}. It simply skips the operation if a different type * of {@link DatasetVersion} is passed. * </p> * {@inheritDoc} * @see org.apache.gobblin.data.management.retention.action.RetentionAction#execute(java.util.List) */ @Override public void execute(List<DatasetVersion> allVersions) throws IOException { // Select version on which access control actions need to performed for (DatasetVersion datasetVersion : this.selectionPolicy.listSelectedVersions(allVersions)) { executeOnVersion(datasetVersion); } }
@Override public Collection<CopyableFile> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { log.info(String.format("Getting copyable files at root path: %s", this.datasetRoot)); List<TimestampedDatasetVersion> versions = Lists.newArrayList(this.datasetVersionFinder.findDatasetVersions(this)); if (versions.isEmpty()) { log.warn("No dataset version can be found. Ignoring."); return Lists.newArrayList(); } Collection<TimestampedDatasetVersion> copyableVersions = this.versionSelectionPolicy.listSelectedVersions(versions); ConcurrentLinkedQueue<CopyableFile> copyableFileList = new ConcurrentLinkedQueue<>(); List<Future<?>> futures = Lists.newArrayList(); for (TimestampedDatasetVersion copyableVersion : copyableVersions) { futures.add(this.executor.submit(this.getCopyableFileGenetator(targetFs, configuration, copyableVersion, copyableFileList))); } try { for (Future<?> future : futures) { future.get(); } } catch (ExecutionException | InterruptedException e) { throw new IOException("Failed to generate copyable files.", e); } finally { ExecutorsUtils.shutdownExecutorService(executor, Optional.of(log)); } return copyableFileList; }
@Override public void clean() throws IOException { List<T> versions = Lists.newArrayList(this.getVersionFinder().findDatasetVersions(this)); Collections.sort(versions, Collections.reverseOrder()); Collection<T> deletableVersions = this.getVersionSelectionPolicy().listSelectedVersions(versions); for (Object version : deletableVersions) { ((DatasetStateStoreVersion) version).getEntry().delete(); } } }
Collection<FileSystemDatasetVersion> versionsSelected = selector.listSelectedVersions(versions);
Collection<HiveDatasetVersion> deletableVersions = this.hiveSelectionPolicy.listSelectedVersions(versions);
Collection<T> deletableVersions = selectionPolicy.listSelectedVersions(versions);