Callable<Path> download(Path path, LocalResource rsrc, UserGroupInformation ugi) throws IOException { DiskChecker.checkDir(new File(path.toUri().getRawPath())); return new FSDownload(lfs, ugi, conf, path, rsrc); }
@VisibleForTesting boolean fileIsPublic(final Path remotePath, FileSystem remoteFs, FileStatus status) throws IOException { return FSDownload.isPublic(remoteFs, remotePath, status, null); }
/** * Returns a boolean to denote whether a cache file is visible to all (public) * or not * * @return true if the path in the current path is visible to all, false * otherwise */ @Private public static boolean isPublic(FileSystem fs, Path current, FileStatus sStat, LoadingCache<Path,Future<FileStatus>> statCache) throws IOException { current = fs.makeQualified(current); //the leaf level file should be readable by others if (!checkPublicPermsForAll(fs, sStat, FsAction.READ_EXECUTE, FsAction.READ)) { return false; } if (Shell.WINDOWS && fs instanceof LocalFileSystem) { // Relax the requirement for public cache on LFS on Windows since default // permissions are "700" all the way up to the drive letter. In this // model, the only requirement for a user is to give EVERYONE group // permission on the file and the file will be considered public. // This code path is only hit when fs.default.name is file:/// (mainly // in tests). return true; } return ancestorsHaveExecutePermissions(fs, current.getParent(), statCache); }
throw new IOException("Invalid resource", e); createDir(destDirPath, cachePerms); final Path dst_work = new Path(destDirPath + "_tmp"); createDir(dst_work, cachePerms); Path dFinal = files.makeQualified(new Path(dst_work, sCopy.getName())); try { Path dTmp = null == userUgi ? files.makeQualified(copy(sCopy, dst_work)) : userUgi.doAs(new PrivilegedExceptionAction<Path>() { public Path run() throws Exception { }; }); unpack(new File(dTmp.toUri()), new File(dFinal.toUri())); changePermissions(dFinal.getFileSystem(conf), dFinal); files.rename(dst_work, destDirPath, Rename.OVERWRITE); } catch (Exception e) {
/** * Localize files. * @param destination destination directory * @throws IOException cannot read or write file * @throws YarnException subcommand returned an error */ private void verifyAndCopy(Path destination) throws IOException, YarnException { final Path sCopy; try { sCopy = resource.getResource().toPath(); } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); } FileSystem sourceFs = sCopy.getFileSystem(conf); FileStatus sStat = sourceFs.getFileStatus(sCopy); if (sStat.getModificationTime() != resource.getTimestamp()) { throw new IOException("Resource " + sCopy + " changed on src filesystem (expected " + resource.getTimestamp() + ", was " + sStat.getModificationTime()); } if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) { if (!isPublic(sourceFs, sCopy, sStat, statCache)) { throw new IOException("Resource " + sCopy + " is not publicly accessible and as such cannot be part of the" + " public cache."); } } downloadAndUnpack(sCopy, destination); }
assumeTrue(FSDownload.ancestorsHaveExecutePermissions(f, basedir, null)); new ConcurrentHashMap<Path,AtomicInteger>(); final CacheLoader<Path,Future<FileStatus>> loader = FSDownload.createStatusCacheLoader(conf); final LoadingCache<Path,Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(new CacheLoader<Path,Future<FileStatus>>() {
srcType, LocalResourceVisibility.PRIVATE, scFileStatus.getLen(), scFileStatus.getModificationTime()); FSDownload download = new FSDownload(localLFS, null, conf, downloadDest, scRsrc, null); try { Path downloaded = download.call();
/** * For each of the requested resources for a container, determines the * appropriate {@link LocalResourcesTracker} and forwards a * {@link LocalResourceRequest} to that tracker. */ private void handleInitContainerResources( ContainerLocalizationRequestEvent rsrcReqs) { Container c = rsrcReqs.getContainer(); // create a loading cache for the file statuses LoadingCache<Path,Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig())); LocalizerContext ctxt = new LocalizerContext( c.getUser(), c.getContainerId(), c.getCredentials(), statCache); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcReqs.getRequestedResources(); for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId() .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); } } }
createDir(destinationTmp, cachePerms); Path dFinal = files.makeQualified(new Path(destinationTmp, sCopy.getName())); try { if (userUgi == null) { verifyAndCopy(dFinal); } else { userUgi.doAs(new PrivilegedExceptionAction<Void>() { changePermissions(dFinal.getFileSystem(conf), dFinal); files.rename(destinationTmp, destDirPath, Rename.OVERWRITE);
private static boolean checkPublicPermsForAll(FileSystem fs, FileStatus status, FsAction dir, FsAction file) throws IOException { FsPermission perms = status.getPermission(); FsAction otherAction = perms.getOtherAction(); if (status.isDirectory()) { if (!otherAction.implies(dir)) { return false; } for (FileStatus child : fs.listStatus(status.getPath())) { if(!checkPublicPermsForAll(fs, child, dir, file)) { return false; } } return true; } return (otherAction.implies(file)); }
/** * Returns true if all ancestors of the specified path have the 'execute' * permission set for all users (i.e. that other users can traverse * the directory hierarchy to the given path) */ @VisibleForTesting static boolean ancestorsHaveExecutePermissions(FileSystem fs, Path path, LoadingCache<Path,Future<FileStatus>> statCache) throws IOException { Path current = path; while (current != null) { //the subdirs in the path should have execute permissions for others if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE, statCache)) { return false; } current = current.getParent(); } return true; }
FileStatus[] statuses = fs.listStatus(path); for (FileStatus status : statuses) { changePermissions(fs, status.getPath());
assumeTrue(FSDownload.ancestorsHaveExecutePermissions(f, basedir, null)); new ConcurrentHashMap<Path,AtomicInteger>(); final CacheLoader<Path,Future<FileStatus>> loader = FSDownload.createStatusCacheLoader(conf); final LoadingCache<Path,Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(new CacheLoader<Path,Future<FileStatus>>() {
throw new IOException("Invalid resource", e); createDir(destDirPath, cachePerms); final Path dst_work = new Path(destDirPath + "_tmp"); createDir(dst_work, cachePerms); Path dFinal = files.makeQualified(new Path(dst_work, sCopy.getName())); try { Path dTmp = null == userUgi ? files.makeQualified(copy(sCopy, dst_work)) : userUgi.doAs(new PrivilegedExceptionAction<Path>() { public Path run() throws Exception { }; }); unpack(new File(dTmp.toUri()), new File(dFinal.toUri())); changePermissions(dFinal.getFileSystem(conf), dFinal); files.rename(dst_work, destDirPath, Rename.OVERWRITE); } catch (Exception e) {
/** * For each of the requested resources for a container, determines the * appropriate {@link LocalResourcesTracker} and forwards a * {@link LocalResourceRequest} to that tracker. */ private void handleInitContainerResources( ContainerLocalizationRequestEvent rsrcReqs) { Container c = rsrcReqs.getContainer(); // create a loading cache for the file statuses LoadingCache<Path,Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig())); LocalizerContext ctxt = new LocalizerContext( c.getUser(), c.getContainerId(), c.getCredentials(), statCache, c.getUserFolder()); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcReqs.getRequestedResources(); for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId() .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); if (LOG.isDebugEnabled()) { LOG.debug("Localizing " + req.getPath() + " for container " + c.getContainerId()); } } } }
private static boolean checkPublicPermsForAll(FileSystem fs, FileStatus status, FsAction dir, FsAction file) throws IOException { FsPermission perms = status.getPermission(); FsAction otherAction = perms.getOtherAction(); if (status.isDirectory()) { if (!otherAction.implies(dir)) { return false; } for (FileStatus child : fs.listStatus(status.getPath())) { if(!checkPublicPermsForAll(fs, child, dir, file)) { return false; } } return true; } return (otherAction.implies(file)); }
/** * Returns true if all ancestors of the specified path have the 'execute' * permission set for all users (i.e. that other users can traverse * the directory hierarchy to the given path) */ @VisibleForTesting static boolean ancestorsHaveExecutePermissions(FileSystem fs, Path path, LoadingCache<Path,Future<FileStatus>> statCache) throws IOException { Path current = path; while (current != null) { //the subdirs in the path should have execute permissions for others if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE, statCache)) { return false; } current = current.getParent(); } return true; }
FileStatus[] statuses = fs.listStatus(path); for (FileStatus status : statuses) { changePermissions(fs, status.getPath());
Callable<Path> download(Path path, LocalResource rsrc, UserGroupInformation ugi) throws IOException { DiskChecker.checkDir(new File(path.toUri().getRawPath())); return new FSDownload(lfs, ugi, conf, path, rsrc); }
@VisibleForTesting boolean fileIsPublic(final Path remotePath, FileSystem remoteFs, FileStatus status) throws IOException { return FSDownload.isPublic(remoteFs, remotePath, status, null); }