/** * Helper method to list out all paths under a specified path. If the {@link org.apache.hadoop.fs.FileSystem} is * unable to list the contents of a relevant directory, will log an error and skip. */ public static List<FileStatus> listPathsRecursively(FileSystem fs, Path path, PathFilter fileFilter) throws IOException { return listPathsRecursivelyHelper(fs, Lists.<FileStatus> newArrayList(), fs.getFileStatus(path), fileFilter); }
private static List<FileStatus> listPathsRecursivelyHelper(FileSystem fs, List<FileStatus> files, FileStatus fileStatus, PathFilter fileFilter) { if (fileFilter.accept(fileStatus.getPath())) { files.add(fileStatus); } if (fileStatus.isDirectory()) { try { for (FileStatus status : fs.listStatus(fileStatus.getPath())) { listPathsRecursivelyHelper(fs, files, status, fileFilter); } } catch (IOException ioe) { LOG.error("Could not list contents of path " + fileStatus.getPath()); } } return files; } }