/** * Returns a reference to the {@link FileSystem} instance for accessing the * file system identified by the given {@link URI}. * * @param uri * the {@link URI} identifying the file system * @return a reference to the {@link FileSystem} instance for accessing the file system identified by the given * {@link URI}. * @throws IOException * thrown if a reference to the file system instance could not be obtained */ public static FileSystem get(URI uri) throws IOException { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(getUnguardedFileSystem(uri)); }
public CompletedFuture(Path entry) { try{ LocalFileSystem fs = (LocalFileSystem) FileSystem.getUnguardedFileSystem(entry.toUri()); result = entry.isAbsolute() ? new Path(entry.toUri().getPath()): new Path(fs.getWorkingDirectory(),entry); } catch (Exception e){ throw new RuntimeException("DistributedCache supports only local files for Collection Environments"); } }
org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(path.toUri()); final FileSystem hadoopFs = (flinkFs instanceof HadoopFileSystem) ? ((HadoopFileSystem) flinkFs).getHadoopFileSystem() : null; org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(genericHdfsUri);
/** * Copies all files from source to target and sets executable flag. Paths might be on different systems. * @param sourcePath source path to copy from * @param targetPath target path to copy to * @param executable if target file should be executable * @throws IOException if the copy fails */ public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { // we unwrap the file system to get raw streams without safety net FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { internalCopyDirectory(sourcePath, targetPath, executable, sFS, tFS); } else { internalCopyFile(sourcePath, targetPath, executable, sFS, tFS); } } }
if (!FileSystem.getUnguardedFileSystem(planPath.toUri()).exists(planPath)) { throw new FileNotFoundException("Plan file " + planFile + " does not exist."); if (!FileSystem.getUnguardedFileSystem(filePath.toUri()).exists(filePath)) { throw new FileNotFoundException("Additional file " + file + " does not exist.");
if (!FileSystem.getUnguardedFileSystem(planPath.toUri()).exists(planPath)) { throw new FileNotFoundException("Plan file " + planFile + " does not exist."); if (!FileSystem.getUnguardedFileSystem(filePath.toUri()).exists(filePath)) { throw new FileNotFoundException("Additional file " + file + " does not exist.");
/** * Returns a reference to the {@link FileSystem} instance for accessing the * file system identified by the given {@link URI}. * * @param uri * the {@link URI} identifying the file system * @return a reference to the {@link FileSystem} instance for accessing the file system identified by the given * {@link URI}. * @throws IOException * thrown if a reference to the file system instance could not be obtained */ public static FileSystem get(URI uri) throws IOException { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(getUnguardedFileSystem(uri)); }
/** * Returns a reference to the {@link FileSystem} instance for accessing the * file system identified by the given {@link URI}. * * @param uri * the {@link URI} identifying the file system * @return a reference to the {@link FileSystem} instance for accessing the file system identified by the given * {@link URI}. * @throws IOException * thrown if a reference to the file system instance could not be obtained */ public static FileSystem get(URI uri) throws IOException { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(getUnguardedFileSystem(uri)); }
public CompletedFuture(Path entry) { try{ LocalFileSystem fs = (LocalFileSystem) FileSystem.getUnguardedFileSystem(entry.toUri()); result = entry.isAbsolute() ? new Path(entry.toUri().getPath()): new Path(fs.getWorkingDirectory(),entry); } catch (Exception e){ throw new RuntimeException("DistributedCache supports only local files for Collection Environments"); } }
public CompletedFuture(Path entry) { try{ LocalFileSystem fs = (LocalFileSystem) FileSystem.getUnguardedFileSystem(entry.toUri()); result = entry.isAbsolute() ? new Path(entry.toUri().getPath()): new Path(fs.getWorkingDirectory(),entry); } catch (Exception e){ throw new RuntimeException("DistributedCache supports only local files for Collection Environments"); } }
/** * Copies all files from source to target and sets executable flag. Paths might be on different systems. * @param sourcePath source path to copy from * @param targetPath target path to copy to * @param executable if target file should be executable * @throws IOException if the copy fails */ public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { // we unwrap the file system to get raw streams without safety net FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { internalCopyDirectory(sourcePath, targetPath, executable, sFS, tFS); } else { internalCopyFile(sourcePath, targetPath, executable, sFS, tFS); } } }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { // TODO rewrite this to make it participate in the closable registry and the lifecycle of a task. // we unwrap the file system to get raw streams without safety net FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) { IOUtils.copyBytes(fsInput, lfsOutput); //noinspection ResultOfMethodCallIgnored new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { LOG.error("could not copy file to local file cache.", ioe); } } } }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { // TODO rewrite this to make it participate in the closable registry and the lifecycle of a task. // we unwrap the file system to get raw streams without safety net FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try (FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath)) { IOUtils.copyBytes(fsInput, lfsOutput); //noinspection ResultOfMethodCallIgnored new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { LOG.error("could not copy file to local file cache.", ioe); } } } }
FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) {