@Override public boolean isDistributedFS() { return originalFs.isDistributedFS(); }
@Override public boolean isDistributedFS() { return unsafeFileSystem.isDistributedFS(); }
private static boolean isOnDistributedFS(final Path path) throws IOException { return path.getFileSystem().isDistributedFS(); }
@Override public void flatMap(FileCopyTask task, Collector<Object> out) throws Exception { LOGGER.info("Processing task: " + task); Path outPath = new Path(targetPath, task.getRelativePath()); FileSystem targetFs = targetPath.getFileSystem(); // creating parent folders in case of a local FS if (!targetFs.isDistributedFS()) { //dealing with cases like file:///tmp or just /tmp File outFile = outPath.toUri().isAbsolute() ? new File(outPath.toUri()) : new File(outPath.toString()); File parentFile = outFile.getParentFile(); if (!parentFile.mkdirs() && !parentFile.exists()) { throw new RuntimeException("Cannot create local file system directories: " + parentFile); } } FSDataOutputStream outputStream = null; FSDataInputStream inputStream = null; try { outputStream = targetFs.create(outPath, FileSystem.WriteMode.OVERWRITE); inputStream = task.getPath().getFileSystem().open(task.getPath()); int bytes = IOUtils.copy(inputStream, outputStream); bytesCounter.add(bytes); } finally { IOUtils.closeQuietly(inputStream); IOUtils.closeQuietly(outputStream); } fileCounter.add(1L); } });
if (!fs.isDistributedFS()) {
if (fs.isDistributedFS()) {
if (!isDistributedFS()) { return false;
if (isDistributedFS()) { return false;
public static void addUserArtifactEntries(Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> userArtifacts, JobGraph jobGraph) { if (!userArtifacts.isEmpty()) { try { java.nio.file.Path tmpDir = Files.createTempDirectory("flink-distributed-cache-" + jobGraph.getJobID()); for (Tuple2<String, DistributedCache.DistributedCacheEntry> originalEntry : userArtifacts) { Path filePath = new Path(originalEntry.f1.filePath); boolean isLocalDir = false; try { FileSystem sourceFs = filePath.getFileSystem(); isLocalDir = !sourceFs.isDistributedFS() && sourceFs.getFileStatus(filePath).isDir(); } catch (IOException ioe) { LOG.warn("Could not determine whether {} denotes a local path.", filePath, ioe); } // zip local directories because we only support file uploads DistributedCache.DistributedCacheEntry entry; if (isLocalDir) { Path zip = FileUtils.compressDirectory(filePath, new Path(tmpDir.toString(), filePath.getName() + ".zip")); entry = new DistributedCache.DistributedCacheEntry(zip.toString(), originalEntry.f1.isExecutable, true); } else { entry = new DistributedCache.DistributedCacheEntry(filePath.toString(), originalEntry.f1.isExecutable, false); } jobGraph.addUserArtifact(originalEntry.f0, entry); } } catch (IOException ioe) { throw new FlinkRuntimeException("Could not compress distributed-cache artifacts.", ioe); } } }
verify(fs).isDistributedFS(); verify(fs).isDistributedFS();
@Override public boolean isDistributedFS() { return unsafeFileSystem.isDistributedFS(); }
@Override public boolean isDistributedFS() { return unsafeFileSystem.isDistributedFS(); }
@Override public boolean isDistributedFS() { return originalFs.isDistributedFS(); }
@Override public boolean isDistributedFS() { return originalFs.isDistributedFS(); }
private static void checkInputParams(ExecutionEnvironment env, Path sourcePath, Path targetPath, int parallelism) throws IOException { if (parallelism <= 0) { throw new IllegalArgumentException("Parallelism should be greater than 0"); } boolean isLocal = env instanceof LocalEnvironment; if (!isLocal && !(sourcePath.getFileSystem().isDistributedFS() && targetPath.getFileSystem().isDistributedFS())) { throw new IllegalArgumentException("In a distributed mode only HDFS input/output paths are supported"); } }
private static boolean isOnDistributedFS(final Path path) throws IOException { return path.getFileSystem().isDistributedFS(); }
private static boolean isOnDistributedFS(final Path path) throws IOException { return path.getFileSystem().isDistributedFS(); }
private static Collection<Tuple2<String, PermanentBlobKey>> uploadUserArtifacts(JobID jobID, Collection<Tuple2<String, Path>> userArtifacts, BlobClient blobClient) throws IOException { Collection<Tuple2<String, PermanentBlobKey>> blobKeys = new ArrayList<>(userArtifacts.size()); for (Tuple2<String, Path> userArtifact : userArtifacts) { // only upload local files if (!userArtifact.f1.getFileSystem().isDistributedFS()) { final PermanentBlobKey blobKey = blobClient.uploadFile(jobID, userArtifact.f1); blobKeys.add(Tuple2.of(userArtifact.f0, blobKey)); } } return blobKeys; }
private static Collection<Tuple2<String, PermanentBlobKey>> uploadUserArtifacts(JobID jobID, Collection<Tuple2<String, Path>> userArtifacts, BlobClient blobClient) throws IOException { Collection<Tuple2<String, PermanentBlobKey>> blobKeys = new ArrayList<>(userArtifacts.size()); for (Tuple2<String, Path> userArtifact : userArtifacts) { // only upload local files if (!userArtifact.f1.getFileSystem().isDistributedFS()) { final PermanentBlobKey blobKey = blobClient.uploadFile(jobID, userArtifact.f1); blobKeys.add(Tuple2.of(userArtifact.f0, blobKey)); } } return blobKeys; }
private static Collection<Tuple2<String, PermanentBlobKey>> uploadUserArtifacts(JobID jobID, Collection<Tuple2<String, Path>> userArtifacts, BlobClient blobClient) throws IOException { Collection<Tuple2<String, PermanentBlobKey>> blobKeys = new ArrayList<>(userArtifacts.size()); for (Tuple2<String, Path> userArtifact : userArtifacts) { // only upload local files if (!userArtifact.f1.getFileSystem().isDistributedFS()) { final PermanentBlobKey blobKey = blobClient.uploadFile(jobID, userArtifact.f1); blobKeys.add(Tuple2.of(userArtifact.f0, blobKey)); } } return blobKeys; }