/** * Get the path to a local file * @return File representing the local path * @throws IllegalArgumentException if this.fs is not the LocalFileSystem */ public File toFile() { if (!(fs instanceof LocalFileSystem)) { throw new IllegalArgumentException("Not a local path: " + path); } return ((LocalFileSystem)fs).pathToFile(path); }
/** * If libjars are set in the conf, parse the libjars. * @param conf * @return libjar urls * @throws IOException */ public static URL[] getLibJars(Configuration conf) throws IOException { String jars = conf.get("tmpjars"); if (jars == null || jars.trim().isEmpty()) { return null; } String[] files = jars.split(","); List<URL> cp = new ArrayList<URL>(); for (String file : files) { Path tmp = new Path(file); if (tmp.getFileSystem(conf).equals(FileSystem.getLocal(conf))) { cp.add(FileSystem.getLocal(conf).pathToFile(tmp).toURI().toURL()); } else { LOG.warn("The libjars file " + tmp + " is not on the local " + "filesystem. It will not be added to the local classpath."); } } return cp.toArray(new URL[0]); }
private static void checkDirInternal(LocalFileSystem localFS, Path dir, FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); }
/** * Renames a src {@link Path} on fs {@link FileSystem} to a dst {@link Path}. If fs is a {@link LocalFileSystem} and * src is a directory then {@link File#renameTo} is called directly to avoid a directory rename race condition where * {@link org.apache.hadoop.fs.RawLocalFileSystem#rename} copies the conflicting src directory into dst resulting in * an extra nested level, such as /root/a/b/c/e/e where e is repeated. * * @param fs the {@link FileSystem} where the src {@link Path} exists * @param src the source {@link Path} which will be renamed * @param dst the {@link Path} to rename to * @return true if rename succeeded, false if rename failed. * @throws IOException if rename failed for reasons other than target exists. */ public static boolean renamePathHandleLocalFSRace(FileSystem fs, Path src, Path dst) throws IOException { if (DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem && fs.isDirectory(src)) { LocalFileSystem localFs = (LocalFileSystem) DecoratorUtils.resolveUnderlyingObject(fs); File srcFile = localFs.pathToFile(src); File dstFile = localFs.pathToFile(dst); return srcFile.renameTo(dstFile); } else { return fs.rename(src, dst); } }
/** * Create the local directory if necessary, also ensure permissions * allow it to be read from and written into. Perform some diskIO * to ensure that the disk is usable for writes. * * @param localFS local filesystem * @param dir directory * @param expected permission * @throws DiskErrorException * @throws IOException */ public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, FsPermission expected) throws DiskErrorException, IOException { checkDirInternal(localFS, dir, expected); doDiskIo(localFS.pathToFile(dir)); }
/** * Create the directory or check permissions if it already exists. * * The semantics of mkdirsWithExistsAndPermissionCheck method is different * from the mkdirs method provided in the Sun's java.io.File class in the * following way: * While creating the non-existent parent directories, this method checks for * the existence of those directories if the mkdir fails at any point (since * that directory might have just been created by some other process). * If both mkdir() and the exists() check fails for any seemingly * non-existent directory, then we signal an error; Sun's mkdir would signal * an error (return false) if a directory it is attempting to create already * exists or the mkdir fails. * * @param localFS local filesystem * @param dir directory to be created or checked * @param expected expected permission * @throws IOException */ static void mkdirsWithExistsAndPermissionCheck( LocalFileSystem localFS, Path dir, FsPermission expected) throws IOException { File directory = localFS.pathToFile(dir); boolean created = false; if (!directory.exists()) created = mkdirsWithExistsCheck(directory); if (created || !localFS.getFileStatus(dir).getPermission().equals(expected)) localFS.setPermission(dir, expected); }
/** * Get the path to a local file * @return File representing the local path * @throws IllegalArgumentException if this.fs is not the LocalFileSystem */ public File toFile() { if (!(fs instanceof LocalFileSystem)) { throw new IllegalArgumentException("Not a local path: " + path); } return ((LocalFileSystem)fs).pathToFile(path); }
/** * Get the path to a local file * @return File representing the local path * @throws IllegalArgumentException if this.fs is not the LocalFileSystem */ public File toFile() { if (!(fs instanceof LocalFileSystem)) { throw new IllegalArgumentException("Not a local path: " + path); } return ((LocalFileSystem)fs).pathToFile(path); }
/** * Get the path to a local file * @return File representing the local path * @throws IllegalArgumentException if this.fs is not the LocalFileSystem */ public File toFile() { if (!(fs instanceof LocalFileSystem)) { throw new IllegalArgumentException("Not a local path: " + path); } return ((LocalFileSystem)fs).pathToFile(path); }
/** * Get the path to a local file * @return File representing the local path * @throws IllegalArgumentException if this.fs is not the LocalFileSystem */ public File toFile() { if (!(fs instanceof LocalFileSystem)) { throw new IllegalArgumentException("Not a local path: " + path); } return ((LocalFileSystem)fs).pathToFile(path); }
/** * If libjars are set in the conf, parse the libjars. */ public static URL[] getLibJars(Configuration conf) throws IOException { String jars = conf.get("tmpjars"); if(jars == null) { return null; } String[] files = jars.split(FILE_SEPERATOR); List<URL> cp = new ArrayList<URL>(); for(String file: files) { Path tmp = new Path(file); if(tmp.getFileSystem(conf).equals(FileSystem.getLocal(conf))) { cp.add(FileSystem.getLocal(conf).pathToFile(tmp).toURI().toURL()); } } return cp.toArray(new URL[0]); }
/** * If libjars are set in the conf, parse the libjars. */ public static URL[] getLibJars(Configuration conf) throws IOException { String jars = conf.get("tmpjars"); if(jars == null) { return null; } String[] files = jars.split(FILE_SEPERATOR); List<URL> cp = new ArrayList<URL>(); for(String file: files) { Path tmp = new Path(file); if(tmp.getFileSystem(conf).equals(FileSystem.getLocal(conf))) { cp.add(FileSystem.getLocal(conf).pathToFile(tmp).toURI().toURL()); } } return cp.toArray(new URL[0]); }
@Override public File getFile() throws IOException { // check for out-of-the-box localFS if (fs instanceof RawLocalFileSystem) { return ((RawLocalFileSystem) fs).pathToFile(path); } if (fs instanceof LocalFileSystem) { return ((LocalFileSystem) fs).pathToFile(path); } throw new UnsupportedOperationException("Cannot resolve File object for " + getDescription()); }
public enum LsOption { Recursive, WithBlockSize };
public enum LsOption { Recursive, WithBlockSize };
@Override public File getFile() throws IOException { // check for out-of-the-box localFS if (this.fs instanceof RawLocalFileSystem) { return ((RawLocalFileSystem) this.fs).pathToFile(this.path); } if (this.fs instanceof LocalFileSystem) { return ((LocalFileSystem) this.fs).pathToFile(this.path); } throw new UnsupportedOperationException("Cannot resolve File object for " + getDescription()); }
/** * Create the local directory if necessary, check permissions and also ensure * it can be read from and written into. * * @param localFS local filesystem * @param dir directory * @param expected permission * @throws DiskErrorException * @throws IOException */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkDirAccess(localFS.pathToFile(dir)); }
/** * Create the local directory if necessary, check permissions and also ensure * it can be read from and written into. * * @param localFS local filesystem * @param dir directory * @param expected permission * @throws DiskErrorException * @throws IOException */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); }
/** * Create the local directory if necessary, check permissions and also ensure * it can be read from and written into. * * @param localFS local filesystem * @param dir directory * @param expected permission * @throws DiskErrorException * @throws IOException */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkDirAccess(localFS.pathToFile(dir)); }
/** * Create the local directory if necessary, check permissions and also ensure * it can be read from and written into. * * @param localFS local filesystem * @param dir directory * @param expected permission * @throws DiskErrorException * @throws IOException */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkDirAccess(localFS.pathToFile(dir)); }