CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).publishDir(new Path("/target")) .preserve(PreserveAttributes.fromMnemonicString("")).build()) .destinationOwnerAndPermission(ownerAndPermission) .ancestorsOwnerAndPermission(ancestorOwnerAndPermissions) .build();
hiveCopyEntityHelper.getConfiguration(), Optional.of(this.partition))) { CopyableFile fileEntity = builder.fileSet(fileSet).checksum(new byte[0]).datasetOutputPath(desiredTargetLocation.location.toString()) .build();
.destination(targetPath) .ancestorsOwnerAndPermission(Lists.<OwnerAndPermission>newArrayList()) // not testing ancestors .build();
hiveCopyEntityHelper.getConfiguration(), Optional.of(this.partition))) { CopyableFile fileEntity = builder.fileSet(fileSet).checksum(new byte[0]).datasetOutputPath(desiredTargetLocation.location.toString()) .build();
Optional.<Partition> absent())) { CopyableFile fileEntity = builder.fileSet(fileSet).datasetOutputPath(desiredTargetLocation.location.toString()).build(); fileEntity.setSourceData(this.helper.getSourceDataset()); fileEntity.setDestinationData(this.helper.getDestinationDataset());
Path thisTargetPath = new Path(configuration.getPublishDir(), filePathRelativeToSearchPath); CopyableFile copyableFile = CopyableFile.fromOriginAndDestination(this.fs, file, thisTargetPath, configuration).fileSet(datasetURN()) .datasetOutputPath(thisTargetPath.toString()).ancestorsOwnerAndPermission(CopyableFile .resolveReplicatedOwnerAndPermissionsRecursively(this.fs, file.getPath().getParent(), nonGlobSearchPath, configuration)).build(); copyableFile.setFsDatasets(this.fs, targetFs); copyableFiles.add(copyableFile);
ancestorsOwnerAndPermission(ancestorOwnerAndPermission));
/** * Get a {@link CopyableFile.Builder}. * * @param originFs {@link FileSystem} where original file exists. * @param origin {@link FileStatus} of the original file. * @param datasetRoot Value of {@link CopyableDataset#datasetRoot} of the dataset creating this {@link CopyableFile}. * @param copyConfiguration {@link CopyConfiguration} for the copy job. * @return a {@link CopyableFile.Builder}. * @deprecated use {@link #fromOriginAndDestination}. This method was changed to remove reliance on dataset root * which is not standard of all datasets. The old functionality on inferring destinations cannot be * achieved without dataset root and common dataset root, so this is an approximation. Copyable datasets * should compute file destinations themselves. */ @Deprecated public static Builder builder(FileSystem originFs, FileStatus origin, Path datasetRoot, CopyConfiguration copyConfiguration) { Path relativePath = PathUtils.relativizePath(origin.getPath(), datasetRoot); Path targetRoot = new Path(copyConfiguration.getPublishDir(), datasetRoot.getName()); Path targetPath = new Path(targetRoot, relativePath); return _hiddenBuilder().originFS(originFs).origin(origin).destination(targetPath) .preserve(copyConfiguration.getPreserve()).configuration(copyConfiguration); }
@Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { List<CopyableFile> copyableFiles = Lists.newArrayList(); for (FileStatus fileStatus : this.files) { if (this.shouldAddToCopyableFiles(fileStatus)) { log.debug("Adding copyable file " + fileStatus.getPath() + "for " + identifier + " in " + this.rootPath); Path targetPath = this.getTargetPath(configuration.getPublishDir(), fileStatus.getPath(), this.identifier); copyableFiles.add(CopyableFile.fromOriginAndDestination(this.fs, fileStatus, targetPath, configuration) .destinationOwnerAndPermission(this.getDestinationOwnerAndPermission()).fileSet(this.fileSet(fileStatus)) .build()); } } return DatasetUtils.instantiateCopyableFileFilter(this.props, this.state, this) .filter(this.fs, targetFs, copyableFiles); }
@VisibleForTesting protected CopyableFile generateCopyableFile(FileStatus singleFile, Path targetPath, long timestampFromPath, Path locationToCopy) throws IOException { return CopyableFile.fromOriginAndDestination(srcFs, singleFile, targetPath, configuration) .originTimestamp(timestampFromPath).upstreamTimestamp(timestampFromPath) .fileSet(PathUtils.getPathWithoutSchemeAndAuthority(locationToCopy).toString()).build(); }
public static Builder fromOriginAndDestination(FileSystem originFs, FileStatus origin, Path destination, CopyConfiguration copyConfiguration) { return _hiddenBuilder().originFS(originFs).origin(origin).destination(destination).configuration(copyConfiguration) .preserve(copyConfiguration.getPreserve()); }
Optional.<Partition> absent())) { CopyableFile fileEntity = builder.fileSet(fileSet).datasetOutputPath(desiredTargetLocation.location.toString()).build(); fileEntity.setSourceData(this.helper.getSourceDataset()); fileEntity.setDestinationData(this.helper.getDestinationDataset());
Path thisTargetPath = new Path(configuration.getPublishDir(), filePathRelativeToSearchPath); CopyableFile copyableFile = CopyableFile.fromOriginAndDestination(this.fs, file, thisTargetPath, configuration).fileSet(datasetURN()) .datasetOutputPath(thisTargetPath.toString()).ancestorsOwnerAndPermission(CopyableFile .resolveReplicatedOwnerAndPermissionsRecursively(this.fs, file.getPath().getParent(), nonGlobSearchPath, configuration)).build(); copyableFile.setFsDatasets(this.fs, targetFs); copyableFiles.add(copyableFile);
ancestorsOwnerAndPermission(ancestorOwnerAndPermission));
.fileSet(fileSet).checksum(checksum) .originTimestamp(originTimestamp).upstreamTimestamp(upstreamTimestamp) .destinationOwnerAndPermission(ownerAndPermission) .origin(origin) .preserve(preserveAttributes) .destination(targetPath) .ancestorsOwnerAndPermission(Lists.<OwnerAndPermission>newArrayList()) .build();
CopyConfiguration.builder(fs, state.getProperties()).preserve(PreserveAttributes.fromMnemonicString("")).build()).build();
/** * Get a {@link CopyableFile.Builder}. * * @param originFs {@link FileSystem} where original file exists. * @param origin {@link FileStatus} of the original file. * @param datasetRoot Value of {@link CopyableDataset#datasetRoot} of the dataset creating this {@link CopyableFile}. * @param copyConfiguration {@link CopyConfiguration} for the copy job. * @return a {@link CopyableFile.Builder}. * @deprecated use {@link #fromOriginAndDestination}. This method was changed to remove reliance on dataset root * which is not standard of all datasets. The old functionality on inferring destinations cannot be * achieved without dataset root and common dataset root, so this is an approximation. Copyable datasets * should compute file destinations themselves. */ @Deprecated public static Builder builder(FileSystem originFs, FileStatus origin, Path datasetRoot, CopyConfiguration copyConfiguration) { Path relativePath = PathUtils.relativizePath(origin.getPath(), datasetRoot); Path targetRoot = new Path(copyConfiguration.getPublishDir(), datasetRoot.getName()); Path targetPath = new Path(targetRoot, relativePath); return _hiddenBuilder().originFS(originFs).origin(origin).destination(targetPath) .preserve(copyConfiguration.getPreserve()).configuration(copyConfiguration); }
public TestDatasetManager(Path testMethodTempPath, State state, String datasetTargetPath, List<String> relativeFilePaths) throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.copyableDataset = new TestCopyableDataset(new Path("origin")); this.metadata = new CopyableDatasetMetadata(this.copyableDataset); this.relativeFilePaths = relativeFilePaths; this.writerOutputPath = new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR)); this.targetPath = new Path(testMethodTempPath, datasetTargetPath); FileStatus file = new FileStatus(0, false, 0, 0, 0, new Path("/file")); FileSystem fs = FileSystem.getLocal(new Configuration()); this.copyEntity = CopyableFile.fromOriginAndDestination(fs, file, new Path("/destination"), CopyConfiguration.builder(fs, state.getProperties()).preserve(PreserveAttributes.fromMnemonicString("")) .build()).build(); fs.mkdirs(testMethodTempPath); log.info("Created a temp directory for test at " + testMethodTempPath); }