@Override public boolean accept(Path path) { for (PathFilter filter : this.pathFilters) { if (!filter.accept(path)) { return false; } } return true; } }
@Override public boolean accept(Path path) { this.pathsProcessed.incrementAndGet(); return this.underlying.accept(path); } }
@Override public boolean accept(Path file) { return dirFilter.accept(file) && !file.getName().toString().startsWith("."); } };
/** * Filter files/directories in the given path using the user-supplied path * filter. Results are added to the given array <code>results</code>. * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ private void listStatus(ArrayList<FileStatus> results, Path f, PathFilter filter) throws FileNotFoundException, IOException { FileStatus listing[] = listStatus(f); Preconditions.checkNotNull(listing, "listStatus should not return NULL"); for (int i = 0; i < listing.length; i++) { if (filter.accept(listing[i].getPath())) { results.add(listing[i]); } } }
private void listStatus(ArrayList<FileStatus> results, Path f, PathFilter filter) throws AccessControlException, FileNotFoundException, IOException { FileStatus[] listing = listStatus(f); if (listing != null) { for (int i = 0; i < listing.length; i++) { if (filter.accept(listing[i].getPath())) { results.add(listing[i]); } } } }
@Override public boolean accept(Path path) { return pattern.matches(path.getName()) && userFilter.accept(path); } }
private static List<FileStatus> listMostNestedPathRecursivelyHelper(FileSystem fs, List<FileStatus> files, FileStatus fileStatus, PathFilter fileFilter) throws IOException { if (fileStatus.isDirectory()) { FileStatus[] curFileStatus = fs.listStatus(fileStatus.getPath()); if (ArrayUtils.isEmpty(curFileStatus)) { files.add(fileStatus); } else { for (FileStatus status : curFileStatus) { listMostNestedPathRecursivelyHelper(fs, files, status, fileFilter); } } } else if (fileFilter.accept(fileStatus.getPath())) { files.add(fileStatus); } return files; }
private static List<FileStatus> listPathsRecursivelyHelper(FileSystem fs, List<FileStatus> files, FileStatus fileStatus, PathFilter fileFilter) { if (fileFilter.accept(fileStatus.getPath())) { files.add(fileStatus); } if (fileStatus.isDirectory()) { try { for (FileStatus status : fs.listStatus(fileStatus.getPath())) { listPathsRecursivelyHelper(fs, files, status, fileFilter); } } catch (IOException ioe) { LOG.error("Could not list contents of path " + fileStatus.getPath()); } } return files; }
/** * A log file has a creation timestamp (in ms) in its file name ({@link #filenum}. This helper * method returns the creation timestamp from a given log file. It extracts the timestamp assuming * the filename is created with the {@link #computeFilename(long filenum)} method. * @return timestamp, as in the log file name. */ protected long getFileNumFromFileName(Path fileName) { checkNotNull(fileName, "file name can't be null"); if (!ourFiles.accept(fileName)) { throw new IllegalArgumentException( "The log file " + fileName + " doesn't belong to this WAL. (" + toString() + ")"); } final String fileNameString = fileName.toString(); String chompedPath = fileNameString.substring(prefixPathStr.length(), (fileNameString.length() - walFileSuffix.length())); return Long.parseLong(chompedPath); }
@Override public boolean accept(Path file) { return dirFilter.accept(file) && !file.getName().startsWith("."); } };
@Override public FileStatus[] listStatus(Path path) throws IOException { try { List<String> fileNames = this.fsHelper.ls(HadoopUtils.toUriPath(path)); List<FileStatus> status = Lists.newArrayListWithCapacity(fileNames.size()); for (String name : fileNames) { Path filePath = new Path(name); if (VALID_PATH_FILTER.accept(filePath)) { status.add(getFileStatus(new Path(path, filePath))); } } return status.toArray(new FileStatus[status.size()]); } catch (FileBasedHelperException e) { throw new IOException(e); } }
public static List<FileStatus> listLocatedStatus(final FileSystem fs, final Path path, final PathFilter filter ) throws IOException { RemoteIterator<LocatedFileStatus> itr = fs.listLocatedStatus(path); List<FileStatus> result = new ArrayList<FileStatus>(); while(itr.hasNext()) { FileStatus stat = itr.next(); if (filter == null || filter.accept(stat.getPath())) { result.add(stat); } } return result; }
/** * Find at most one *.properties file in the input {@link Path} and load it using fallback as fallback. * @return The {@link Config} in path with sysProps as fallback. * @throws IOException */ private Config findAndLoadGlobalConfigInDirectory(Path path, Config fallback) throws IOException { FileStatus[] files = this.fs.listStatus(path, GLOBAL_PATH_FILTER); if (files == null) { log.warn("Could not list files at path " + path); return ConfigFactory.empty(); } if (files.length > 1) { throw new IOException("Found more than one global properties file at path " + path); } if (files.length == 0) { return fallback; } if (GLOBAL_HOCON_PATH_FILTER.accept(files[0].getPath())) { return loadHoconConfigWithFallback(files[0].getPath(), fallback); } else if (GLOBAL_PROPS_PATH_FILTER.accept(files[0].getPath())) { return loadJavaPropsWithFallback(files[0].getPath(), fallback); } else { throw new IllegalStateException("Unsupported global configuration file: " + files[0].getPath()); } }
public static List<FileStatus> listLocatedStatus(final FileSystem fs, final Path path, final PathFilter filter ) throws IOException { RemoteIterator<LocatedFileStatus> itr = fs.listLocatedStatus(path); List<FileStatus> result = new ArrayList<FileStatus>(); while(itr.hasNext()) { FileStatus stat = itr.next(); if (filter == null || filter.accept(stat.getPath())) { result.add(stat); } } return result; }
void findNext() throws IOException { while (iter.hasNext()) { LocatedFileStatus status = iter.next(); if (filter.accept(status.getPath())) { nextFile = status; return; } } // No more matching files in the iterator nextFile = null; } }
private boolean toProcessEvent(ProcessContext context, Event event) { final String[] eventTypes = context.getProperty(EVENT_TYPES).getValue().split(","); for (String name : eventTypes) { if (name.trim().equalsIgnoreCase(event.getEventType().name())) { return notificationConfig.getPathFilter().accept(new Path(getPath(event))); } } return false; }
@Test public void testDeleteEventDeltaDirPathFilter() throws Exception { Path positivePath = new Path("delete_delta_000001_000010"); Path negativePath = new Path("delta_000001_000010"); assertEquals(true, AcidUtils.deleteEventDeltaDirFilter.accept(positivePath)); assertEquals(false, AcidUtils.deleteEventDeltaDirFilter.accept(negativePath)); }
public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files, PathFilter filter) throws IOException { RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.isDirectory()) { continue; } // apply filter if (filter.accept(lfs.getPath())) { files.add(lfs.getPath().toString()); } } return files; }
@Override public List<HdfsFileStatusWithId> listLocatedHdfsStatus( FileSystem fs, Path p, PathFilter filter) throws IOException { DistributedFileSystem dfs = ensureDfs(fs); DFSClient dfsc = dfs.getClient(); final String src = p.toUri().getPath(); DirectoryListing current = dfsc.listPaths(src, org.apache.hadoop.hdfs.protocol.HdfsFileStatus.EMPTY_NAME, true); if (current == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } final URI fsUri = fs.getUri(); List<HdfsFileStatusWithId> result = new ArrayList<HdfsFileStatusWithId>( current.getPartialListing().length); while (current != null) { org.apache.hadoop.hdfs.protocol.HdfsFileStatus[] hfss = current.getPartialListing(); for (int i = 0; i < hfss.length; ++i) { HdfsLocatedFileStatus next = (HdfsLocatedFileStatus)(hfss[i]); if (filter != null) { Path filterPath = next.getFullPath(p).makeQualified(fsUri, null); if (!filter.accept(filterPath)) continue; } LocatedFileStatus lfs = next.makeQualifiedLocated(fsUri, p); result.add(new HdfsFileStatusWithIdImpl(lfs, next.getFileId())); } current = current.hasMore() ? dfsc.listPaths(src, current.getLastName(), true) : null; } return result; }
/** * Will assert that actual files match expected. * @param expectedFiles - suffixes of expected Paths. Must be the same length * @param rootPath - table or partition root where to start looking for actual files, recursively */ void assertExpectedFileSet(Set<String> expectedFiles, String rootPath) throws Exception { int suffixLength = 0; for(String s : expectedFiles) { if(suffixLength > 0) { assert suffixLength == s.length() : "all entries must be the same length. current: " + s; } suffixLength = s.length(); } FileSystem fs = FileSystem.get(hiveConf); Set<String> actualFiles = new HashSet<>(); RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(new Path(rootPath), true); while (remoteIterator.hasNext()) { LocatedFileStatus lfs = remoteIterator.next(); if(!lfs.isDirectory() && org.apache.hadoop.hive.common.FileUtils.HIDDEN_FILES_PATH_FILTER.accept(lfs.getPath())) { String p = lfs.getPath().toString(); actualFiles.add(p.substring(p.length() - suffixLength, p.length())); } } Assert.assertEquals("Unexpected file list", expectedFiles, actualFiles); } void checkExpected(List<String> rs, String[][] expected, String msg, Logger LOG, boolean checkFileName) {