@Override public boolean isFile() { return status.isFile(); }
@Override public boolean isFile() { return realStatus.isFile(); }
@Override public boolean isFile() { return myFs.isFile(); }
private void fsMove(Path from, Path to) throws IOException { for (FileStatus fileStatus : this.datasetOwnerFs.listStatus(from)) { if (fileStatus.isFile()) { this.datasetOwnerFs.rename(fileStatus.getPath(), to); } } } }
/** True iff the named path is a regular file. * Note: Avoid using this method. Instead reuse the FileStatus * returned by {@link #getFileStatus(Path)} or listStatus() methods. * * @param f path to check * @throws IOException IO failure * @deprecated Use {@link #getFileStatus(Path)} instead */ @Deprecated public boolean isFile(Path f) throws IOException { try { return getFileStatus(f).isFile(); } catch (FileNotFoundException e) { return false; // f does not exist } }
private URI mostRecentInDir(final Path dir, final Pattern pattern) throws IOException { final PathFilter filter = new PathFilter() { @Override public boolean accept(Path path) { return pattern == null || pattern.matcher(path.getName()).matches(); } }; long modifiedTime = Long.MIN_VALUE; URI mostRecentURI = null; final FileSystem fs = dir.getFileSystem(config); for (FileStatus status : fs.listStatus(dir, filter)) { if (status.isFile()) { final long thisModifiedTime = status.getModificationTime(); if (thisModifiedTime >= modifiedTime) { modifiedTime = thisModifiedTime; mostRecentURI = status.getPath().toUri(); } } } return mostRecentURI; }
@Override protected List<FileStatus> listStatus(JobContext job) throws IOException { List<FileStatus> result = super.listStatus(job); Iterator<FileStatus> it = result.iterator(); while (it.hasNext()) { FileStatus stat = it.next(); if (!stat.isFile() || (stat.getLen() == 0 && !stat.getPath().toUri().getScheme().equals("nullscan"))) { it.remove(); } } return result; } };
private void fsMove(Path from, Path to) throws IOException { if (PartitionUtils.isUnixTimeStamp(from.getName())) { this.versionOwnerFs.rename(from, to.getParent()); } else { for (FileStatus fileStatus : this.versionOwnerFs.listStatus(from)) { if (fileStatus.isFile()) { this.versionOwnerFs.rename(fileStatus.getPath(), to); } } } } }
public static Pair<Long, Long> getSizeAndLastModified(String path) throws IOException { FileSystem fs = HadoopUtil.getFileSystem(path); // get all contained files if path is directory ArrayList<FileStatus> allFiles = new ArrayList<>(); FileStatus status = fs.getFileStatus(new Path(path)); if (status.isFile()) { allFiles.add(status); } else { FileStatus[] listStatus = fs.listStatus(new Path(path)); allFiles.addAll(Arrays.asList(listStatus)); } long size = 0; long lastModified = 0; for (FileStatus file : allFiles) { size += file.getLen(); lastModified = Math.max(lastModified, file.getModificationTime()); } return Pair.newPair(size, lastModified); }
/** * Computes the size of a single file on the filesystem. If the size cannot be computed for some * reason, a {@link SpaceLimitingException} is thrown, as the file may violate a quota. If the * provided path does not reference a file, an {@link IllegalArgumentException} is thrown. * * @param fs The FileSystem which the path refers to a file upon * @param path The path on the {@code fs} to a file whose size is being checked * @return The size in bytes of the file */ long getFileSize(FileSystem fs, String path) throws SpaceLimitingException { final FileStatus status; try { status = fs.getFileStatus(new Path(Objects.requireNonNull(path))); } catch (IOException e) { throw new SpaceLimitingException( getPolicyName(), "Could not verify length of file to bulk load: " + path, e); } if (!status.isFile()) { throw new IllegalArgumentException(path + " is not a file."); } return status.getLen(); } }
@Override public LocatedFileStatus next() throws IOException { if (!hasNext()) { throw new NoSuchElementException("No more entries in " + f); } FileStatus result = stats[i++]; // for files, use getBlockLocations(FileStatus, int, int) to avoid // calling getFileStatus(Path) to load the FileStatus again BlockLocation[] locs = result.isFile() ? getFileBlockLocations(result, 0, result.getLen()) : null; return new LocatedFileStatus(result, locs); } };
/** * Convenience method, so that we don't open a new connection when using this * method from within another method. Otherwise every API invocation incurs * the overhead of opening/closing a TCP connection. */ private boolean isFile(FTPClient client, Path file) { try { return getFileStatus(client, file).isFile(); } catch (FileNotFoundException e) { return false; // file does not exist } catch (IOException ioe) { throw new FTPException("File check failed", ioe); } }
@Override public LocatedFileStatus next() throws IOException { if (!hasNext()) { throw new NoSuchElementException("No more entry in " + f); } FileStatus result = itor.next(); BlockLocation[] locs = null; if (result.isFile()) { locs = getFileBlockLocations( result.getPath(), 0, result.getLen()); } return new LocatedFileStatus(result, locs); } };
protected Set<String> listAllDataFiles(HdfsContext context, Path path) throws IOException { Set<String> result = new HashSet<>(); FileSystem fileSystem = hdfsEnvironment.getFileSystem(context, path); if (fileSystem.exists(path)) { for (FileStatus fileStatus : fileSystem.listStatus(path)) { if (fileStatus.getPath().getName().startsWith(".presto")) { // skip hidden files } else if (fileStatus.isFile()) { result.add(fileStatus.getPath().toString()); } else if (fileStatus.isDirectory()) { result.addAll(listAllDataFiles(context, fileStatus.getPath())); } } } return result; }
private void dumpLogDirState() { try { FileStatus[] files = fs.listStatus(logDir); if (files != null && files.length > 0) { for (FileStatus file: files) { assertTrue(file.toString(), file.isFile()); LOG.debug("log file " + file.getPath() + " size=" + file.getLen()); } } else { LOG.debug("no files under: " + logDir); } } catch (IOException e) { LOG.warn("Unable to dump " + logDir, e); } }
@Test(expected = IllegalArgumentException.class) public void testFileIsNotAFile() throws Exception { final List<String> paths = new ArrayList<>(); String path = "/1"; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(1000L); when(status.isFile()).thenReturn(false); paths.add(path); // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE); policy.initialize(rss, tableName, snapshot); // If the file to bulk load isn't a file, this should throw an exception policy.computeBulkLoadSize(fs, paths); }
@Test public void testFilesUnderLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 100L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); }
@Test(expected = SpaceLimitingException.class) public void testOneFileInBatchOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1000L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); }
@Test(expected = SpaceLimitingException.class) public void testSumOfFilesOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now, but 5*1024 files would push us to violation SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 5000L); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); } }
/** * Called with a source and target destination pair * @param src for the operation * @param dst for the operation * @throws IOException if anything goes wrong */ protected void processPath(PathData src, PathData dst) throws IOException { if (src.stat.isSymlink()) { // TODO: remove when FileContext is supported, this needs to either // copy the symlink or deref the symlink throw new PathOperationException(src.toString()); } else if (src.stat.isFile()) { copyFileToTarget(src, dst); } else if (src.stat.isDirectory() && !isRecursive()) { throw new PathIsDirectoryException(src.toString()); } }