protected List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits List<FileStatus> files = new ArrayList<>(); for (Path filePath: getFilePaths()) { final FileSystem fs = filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(filePath); for (FileStatus partial : partials) { if (!partial.isDir()) { files.add(partial); } } } else { files.add(pathFile); } } return files; }
private static void addToZip(Path fileOrDirectory, FileSystem fs, Path rootDir, ZipOutputStream out) throws IOException { String relativePath = fileOrDirectory.getPath().replace(rootDir.getPath() + '/', ""); if (fs.getFileStatus(fileOrDirectory).isDir()) { out.putNextEntry(new ZipEntry(relativePath + '/')); for (FileStatus containedFile : fs.listStatus(fileOrDirectory)) { addToZip(containedFile.getPath(), fs, rootDir, out); } } else { ZipEntry entry = new ZipEntry(relativePath); out.putNextEntry(entry); try (FSDataInputStream in = fs.open(fileOrDirectory)) { IOUtils.copyBytes(in, out, false); } out.closeEntry(); } }
private static void internalCopyDirectory(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } }
public VirtualFileServerHandler(Path path) throws IOException { this.path = path; if (!path.isAbsolute()) { throw new IllegalArgumentException("path must be absolute: " + path.toString()); } this.fs = path.getFileSystem(); if (!fs.exists(path) || fs.getFileStatus(path).isDir()) { throw new IllegalArgumentException("no such file: " + path.toString()); } }
if (!status.isDir()) { Path filePath = status.getPath(); long modificationTime = status.getModificationTime();
if (dir.isDir()) { if (acceptFile(dir) && enumerateNestedFiles) { length += addFilesInDir(dir.getPath(), files, logExcludedFiles);
protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path filePath, FileSystem fs, ArrayList<FileStatus> files) throws IOException { // get the file info and check whether the cached statistics are still valid. final FileStatus file = fs.getFileStatus(filePath); long totalLength = 0; // enumerate all files if (file.isDir()) { totalLength += addFilesInDir(file.getPath(), files, false); } else { files.add(file); testForUnsplittable(file); totalLength += file.getLen(); } // check the modification time stamp long latestModTime = 0; for (FileStatus f : files) { latestModTime = Math.max(f.getModificationTime(), latestModTime); } // check whether the cached statistics are still valid, if we have any if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) { return cachedStats; } // sanity check if (totalLength <= 0) { totalLength = BaseStatistics.SIZE_UNKNOWN; } return new FileBaseStatistics(latestModTime, totalLength, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN); }
return exists(outPath) && getFileStatus(outPath).isDir();
private static void getCopyTasks(Path p, String rel, List<FileCopyTask> tasks) throws IOException { FileStatus[] res = p.getFileSystem().listStatus(p); if (res == null) { return; } for (FileStatus fs : res) { if (fs.isDir()) { getCopyTasks(fs.getPath(), rel + fs.getPath().getName() + "/", tasks); } else { Path cp = fs.getPath(); tasks.add(new FileCopyTask(cp, rel + cp.getName())); } } } }
public static void addUserArtifactEntries(Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> userArtifacts, JobGraph jobGraph) { if (!userArtifacts.isEmpty()) { try { java.nio.file.Path tmpDir = Files.createTempDirectory("flink-distributed-cache-" + jobGraph.getJobID()); for (Tuple2<String, DistributedCache.DistributedCacheEntry> originalEntry : userArtifacts) { Path filePath = new Path(originalEntry.f1.filePath); boolean isLocalDir = false; try { FileSystem sourceFs = filePath.getFileSystem(); isLocalDir = !sourceFs.isDistributedFS() && sourceFs.getFileStatus(filePath).isDir(); } catch (IOException ioe) { LOG.warn("Could not determine whether {} denotes a local path.", filePath, ioe); } // zip local directories because we only support file uploads DistributedCache.DistributedCacheEntry entry; if (isLocalDir) { Path zip = FileUtils.compressDirectory(filePath, new Path(tmpDir.toString(), filePath.getName() + ".zip")); entry = new DistributedCache.DistributedCacheEntry(zip.toString(), originalEntry.f1.isExecutable, true); } else { entry = new DistributedCache.DistributedCacheEntry(filePath.toString(), originalEntry.f1.isExecutable, false); } jobGraph.addUserArtifact(originalEntry.f0, entry); } } catch (IOException ioe) { throw new FlinkRuntimeException("Could not compress distributed-cache artifacts.", ioe); } } }
final FileStatus pathFile = fs.getFileStatus(path); if (pathFile.isDir()) { totalLength += addFilesInDir(path, files, true); } else {
/** * Copies all files from source to target and sets executable flag. Paths might be on different systems. * @param sourcePath source path to copy from * @param targetPath target path to copy to * @param executable if target file should be executable * @throws IOException if the copy fails */ public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { // we unwrap the file system to get raw streams without safety net FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { internalCopyDirectory(sourcePath, targetPath, executable, sFS, tFS); } else { internalCopyFile(sourcePath, targetPath, executable, sFS, tFS); } } }
assertFalse(status.isDir());
assertTrue(localstatus1.isDir());
private static void assertState(DistributedCache.DistributedCacheEntry entry, boolean isExecutable, boolean isZipped) throws IOException { assertNotNull(entry); assertEquals(isExecutable, entry.isExecutable); assertEquals(isZipped, entry.isZipped); org.apache.flink.core.fs.Path filePath = new org.apache.flink.core.fs.Path(entry.filePath); assertTrue(filePath.getFileSystem().exists(filePath)); assertFalse(filePath.getFileSystem().getFileStatus(filePath).isDir()); } }
@Override public boolean isDir() { return fileStatus.isDir(); }
private static void getCopyTasks(Path p, String rel, List<FileCopyTask> tasks) throws IOException { FileStatus[] res = p.getFileSystem().listStatus(p); if (res == null) { return; } for (FileStatus fs : res) { if (fs.isDir()) { getCopyTasks(fs.getPath(), rel + fs.getPath().getName() + "/", tasks); } else { Path cp = fs.getPath(); tasks.add(new FileCopyTask(cp, rel + cp.getName())); } } }