/** * Helper method to list out all files under a specified path. The specified {@link PathFilter} is treated as a file * filter, that is it is only applied to file {@link Path}s. */ public static List<FileStatus> listFilesRecursively(FileSystem fs, Path path, PathFilter fileFilter) throws IOException { return listFilesRecursively(fs, path, fileFilter, false); }
/** * Given a path to copy, list all files rooted at the given path to copy * * @param fs the file system of the path * @param path root path to copy * @param fileFilter a filter only applied to root * @param includeEmptyDirectories a control to include empty directories for copy */ public static List<FileStatus> listFilesToCopyAtPath(FileSystem fs, Path path, PathFilter fileFilter, boolean includeEmptyDirectories) throws IOException { return listFilesToCopyAtPath(fs, path, fileFilter, false, includeEmptyDirectories); }
/** * Method to list out all files, or directory if no file exists, under a specified path. */ public static List<FileStatus> listMostNestedPathRecursively(FileSystem fs, Path path) throws IOException { return listMostNestedPathRecursively(fs, path, NO_OP_PATH_FILTER); }
/** * Sets the {@link FsPermission}, owner, group for the path passed. And recursively to all directories and files under * it. */ private void setRecursivePermission(Path path, OwnerAndPermission ownerAndPermission) throws IOException { List<FileStatus> files = FileListUtils.listPathsRecursively(this.fs, path, FileListUtils.NO_OP_PATH_FILTER); // Set permissions bottom up. Permissions are set to files first and then directories Collections.reverse(files); for (FileStatus file : files) { safeSetPathPermission(file.getPath(), addExecutePermissionsIfRequired(file, ownerAndPermission)); } }
/** * Helper method to list out all files under a specified path. If applyFilterToDirectories is false, the supplied * {@link PathFilter} will only be applied to files. */ public static List<FileStatus> listFilesRecursively(FileSystem fs, Path path, PathFilter fileFilter, boolean applyFilterToDirectories) throws IOException { return listFilesRecursivelyHelper(fs, Lists.newArrayList(), fs.getFileStatus(path), fileFilter, applyFilterToDirectories, false); }
/** * Method to list out all files, or directory if no file exists, under a specified path. * The specified {@link PathFilter} is treated as a file filter, that is it is only applied to file {@link Path}s. */ public static List<FileStatus> listMostNestedPathRecursively(FileSystem fs, Path path, PathFilter fileFilter) throws IOException { return listMostNestedPathRecursivelyHelper(fs, Lists.<FileStatus>newArrayList(), fs.getFileStatus(path), fileFilter); }
localFs.mkdirs(subDir); localFs.create(new Path(subDir, TEST_FILE_NAME2)); List<FileStatus> testFiles = FileListUtils.listPathsRecursively(localFs, baseDir, new PathFilter() { @Override public boolean accept(Path path) {
/** * Given a path to copy, list all files rooted at the given path to copy * * @param fs the file system of the path * @param path root path to copy * @param fileFilter a filter only applied to root * @param applyFilterToDirectories a control to decide whether to apply filter to directories * @param includeEmptyDirectories a control to include empty directories for copy */ public static List<FileStatus> listFilesToCopyAtPath(FileSystem fs, Path path, PathFilter fileFilter, boolean applyFilterToDirectories, boolean includeEmptyDirectories) throws IOException { List<FileStatus> files = Lists.newArrayList(); FileStatus rootFile = fs.getFileStatus(path); listFilesRecursivelyHelper(fs, files, rootFile, fileFilter, applyFilterToDirectories, includeEmptyDirectories); // Copy the empty root directory if (files.size() == 0 && rootFile.isDirectory() && includeEmptyDirectories) { files.add(rootFile); } return files; }
private static List<FileStatus> listMostNestedPathRecursivelyHelper(FileSystem fs, List<FileStatus> files, FileStatus fileStatus, PathFilter fileFilter) throws IOException { if (fileStatus.isDirectory()) { FileStatus[] curFileStatus = fs.listStatus(fileStatus.getPath()); if (ArrayUtils.isEmpty(curFileStatus)) { files.add(fileStatus); } else { for (FileStatus status : curFileStatus) { listMostNestedPathRecursivelyHelper(fs, files, status, fileFilter); } } } else if (fileFilter.accept(fileStatus.getPath())) { files.add(fileStatus); } return files; }
public static List<FileStatus> listFilesRecursively(FileSystem fs, Path path) throws IOException { return listFilesRecursively(fs, path, NO_OP_PATH_FILTER); }
public static List<FileStatus> listMostNestedPathRecursively(FileSystem fs, Iterable<Path> paths) throws IOException { List<FileStatus> results = Lists.newArrayList(); for (Path path : paths) { results.addAll(listMostNestedPathRecursively(fs, path)); } return results; }
@VisibleForTesting protected List<FileStatus> getFilesAtPath(FileSystem fs, Path path, PathFilter fileFilter) throws IOException { try { return FileListUtils .listFilesToCopyAtPath(fs, path, fileFilter, applyFilterToDirectories, includeEmptyDirectories); } catch (FileNotFoundException fnfe) { return Lists.newArrayList(); } }
for (FileStatus fileStatus : FileListUtils.listPathsRecursively(this.fs, hdfsNewVersionPath, FileListUtils.NO_OP_PATH_FILTER)) { this.fs.setPermission(fileStatus.getPath(), deploymentConfig.getStorePermissions());
private static List<FileStatus> listFilesRecursivelyHelper(FileSystem fs, List<FileStatus> files, FileStatus fileStatus, PathFilter fileFilter, boolean applyFilterToDirectories, boolean includeEmptyDirectories) throws FileNotFoundException, IOException { if (fileStatus.isDirectory()) { for (FileStatus status : fs .listStatus(fileStatus.getPath(), applyFilterToDirectories ? fileFilter : NO_OP_PATH_FILTER)) { if (status.isDirectory()) { // Number of files collected before diving into the directory int numFilesBefore = files.size(); listFilesRecursivelyHelper(fs, files, status, fileFilter, applyFilterToDirectories, includeEmptyDirectories); // Number of files collected after diving into the directory int numFilesAfter = files.size(); if (numFilesAfter == numFilesBefore && includeEmptyDirectories) { /* * This is effectively an empty directory, which needs explicit copying. Has there any data file * in the directory, the directory would be created as a side-effect of copying the data file */ files.add(status); } } else { listFilesRecursivelyHelper(fs, files, status, fileFilter, applyFilterToDirectories, includeEmptyDirectories); } } } else if (fileFilter.accept(fileStatus.getPath())) { files.add(fileStatus); } return files; }
/** * Method to list out all files, or directory if no file exists, under a specified path. * The specified {@link PathFilter} is treated as a file filter, that is it is only applied to file {@link Path}s. */ public static List<FileStatus> listMostNestedPathRecursively(FileSystem fs, Path path, PathFilter fileFilter) throws IOException { return listMostNestedPathRecursivelyHelper(fs, Lists.<FileStatus>newArrayList(), fs.getFileStatus(path), fileFilter); }
public static List<FileStatus> listFilesRecursively(FileSystem fs, Iterable<Path> paths) throws IOException { List<FileStatus> results = Lists.newArrayList(); for (Path path : paths) { results.addAll(listFilesRecursively(fs, path)); } return results; }
private Optional<Long> getLatestModTime(Iterable<Path> paths) throws IOException { long latestModTime = Long.MIN_VALUE; for (FileStatus status : FileListUtils.listMostNestedPathRecursively(this.fs, paths)) { latestModTime = Math.max(latestModTime, status.getModificationTime()); } return latestModTime == Long.MIN_VALUE ? Optional.<Long> absent() : Optional.of(latestModTime); }
public Set<FileStatus> getPaths(boolean skipHiddenPaths) throws IOException { if (!this.fs.exists(this.rootPath)) { return Sets.newHashSet(); } PathFilter actualFilter = skipHiddenPaths ? new AndPathFilter(new HiddenFilter(), this.pathFilter) : this.pathFilter; List<FileStatus> files = FileListUtils.listFilesToCopyAtPath(this.fs, this.rootPath, actualFilter, includeEmptyDirectories); return Sets.newHashSet(files); } }
/** * Sets the {@link FsPermission}, owner, group for the path passed. And recursively to all directories and files under * it. */ private void setRecursivePermission(Path path, OwnerAndPermission ownerAndPermission) throws IOException { List<FileStatus> files = FileListUtils.listPathsRecursively(this.fs, path, FileListUtils.NO_OP_PATH_FILTER); // Set permissions bottom up. Permissions are set to files first and then directories Collections.reverse(files); for (FileStatus file : files) { safeSetPathPermission(file.getPath(), addExecutePermissionsIfRequired(file, ownerAndPermission)); } }
/** * Helper method to list out all files under a specified path. If applyFilterToDirectories is false, the supplied * {@link PathFilter} will only be applied to files. */ public static List<FileStatus> listFilesRecursively(FileSystem fs, Path path, PathFilter fileFilter, boolean applyFilterToDirectories) throws IOException { return listFilesRecursivelyHelper(fs, Lists.newArrayList(), fs.getFileStatus(path), fileFilter, applyFilterToDirectories, false); }