/** * Fixes the path if it denotes a local (relative) file without the proper protocol prefix. */ private static Path normalizePath(Path path) { URI uri = path.toUri(); if (uri.getScheme() == null) { try { uri = new URI("file", uri.getHost(), uri.getPath(), uri.getFragment()); path = new Path(uri.toString()); } catch (URISyntaxException e) { throw new IllegalArgumentException("path is invalid", e); } } return path; } }
private File pathToFile(Path path) { if (!path.isAbsolute()) { path = new Path(getWorkingDirectory(), path); } return new File(path.toUri().getPath()); }
/** * Resolve a child path against a parent path. * * @param parent * the parent path * @param child * the child path */ public Path(Path parent, String child) { this(parent, new Path(child)); }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { FileSystem sFS = sourcePath.getFileSystem(); FileSystem tFS = targetPath.getFileSystem(); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try { FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath); IOUtils.copyBytes(fsInput, lfsOutput); new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { } } } }
public Path call() { Path tmp = getTempDir(jobID, name); try { if (!lfs.exists(tmp)) { FSDataOutputStream lfsOutput = lfs.create(tmp, false); Path distributedPath = new Path(filePath); FileSystem fs = distributedPath.getFileSystem(); FSDataInputStream fsInput = fs.open(distributedPath); IOUtils.copyBytes(fsInput, lfsOutput); } } catch (IOException e1) { throw new RuntimeException("Error copying a file from hdfs to the local fs", e1); } return tmp; } }
/** * Returns the FileSystem that owns this Path. * * @return the FileSystem that owns this Path * @throws IOException * thrown if the file system could not be retrieved */ public FileSystem getFileSystem() throws IOException { return FileSystem.get(this.toUri()); }
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.outputFilePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockBasedInput = new BlockBasedOutput(this.stream, (int) blockSize); this.dataOutputStream = new DataOutputStream(this.blockBasedInput); }
public String toString() { return this.filePath == null ? "File Input (unknown file)" : "File Input (" + this.filePath.toString() + ')'; }
@Override public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException { // Check if the user has specified a path if (this.path == null) { throw new IllegalConfigurationException(this.getName() + " does not specify an input path"); } // Check if the path is valid try { final FileSystem fs = this.path.getFileSystem(); final FileStatus f = fs.getFileStatus(this.path); if (f == null) { throw new IOException(this.path.toString() + " led to a null object"); } } catch (IOException e) { throw new IllegalConfigurationException("Cannot access file or directory: " + StringUtils.stringifyException(e)); } // register the path in the configuration invokable.getTaskConfiguration() .setString(AbstractFileInputTask.INPUT_PATH_CONFIG_KEY, this.path.toString()); // Finally, see if the task itself has a valid configuration super.checkConfiguration(invokable); }
private boolean testForUnsplittable(FileStatus pathFile) { if(pathFile.getPath().getName().endsWith(DEFLATE_SUFFIX)) { unsplittable = true; return true; } return false; }
@Override public FSDataOutputStream create(final Path f, final boolean overwrite, final int bufferSize, final short replication, final long blockSize) throws IOException { if (exists(f) && !overwrite) { throw new IOException("File already exists:" + f); } final Path parent = f.getParent(); if (parent != null && !mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent.toString()); } final File file = pathToFile(f); return new LocalDataOutputStream(file); }
/** * {@inheritDoc} */ @Override public boolean equals(final Object obj) { if (obj == null) { return false; } if (this.jobID == null) { return false; } if (this.clientPath == null) { return false; } if (obj instanceof LibraryTranslationKey) { final LibraryTranslationKey key = (LibraryTranslationKey) obj; if (this.jobID.equals(key.getJobID()) && this.clientPath.equals(key.getClientPath())) { return true; } } return false; }
/** * Resolve a child path against a parent path. * * @param parent * the parent path * @param child * the child path */ public Path(String parent, Path child) { this(new Path(parent), child); }
private File pathToFile(Path path) { if (!path.isAbsolute()) { path = new Path(getWorkingDirectory(), path); } return new File(path.toUri().getPath()); }
@Override public boolean preVisit(final Operator visitable) { int degree = GenericTestPlan.this.getDegreeOfParallelism(); if (visitable instanceof GenericDataSource<?>) degree = 1; else if (degree > 1 && visitable instanceof FileDataSink) try { final Path path = new Path(((FileDataSink) visitable).getFilePath()); final FileSystem fs = path.getFileSystem(); final FileStatus f = fs.getFileStatus(path); if (!f.isDir()) { fs.delete(path, false); fs.mkdirs(path); } } catch (final IOException e) { e.printStackTrace(); } if (visitable.getDegreeOfParallelism() == -1) visitable.setDegreeOfParallelism(degree); return true; } });
/** * Returns the FileSystem that owns this Path. * * @return the FileSystem that owns this Path * @throws IOException * thrown if the file system could not be retrieved */ public FileSystem getFileSystem() throws IOException { return FileSystem.get(this.toUri()); }
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.outputFilePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockBasedInput = new BlockBasedOutput(this.stream, (int) blockSize); this.dataOutputStream = new DataOutputStream(this.blockBasedInput); }
public String toString() { return this.filePath == null ? "File Input (unknown file)" : "File Input (" + this.filePath.toString() + ')'; }