Refine search
public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(path); localResource.setResource(ConverterUtils.getYarnUrlFromPath(path)); localResource.setSize(fileStatus.getLen()); localResource.setTimestamp(fileStatus.getModificationTime()); localResource.setType(type); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); }
assertFalse(targetFileSystem.exists(targetDir)); srcPath = new Path("file://" + srcDir.getAbsolutePath()); } else { srcPath = new Path(srcDir.getAbsolutePath()); HashMap<String, LocalResource> localResources = new HashMap<>(); AbstractYarnClusterDescriptor.uploadAndRegisterFiles( Collections.singletonList(new File(srcPath.toUri().getPath())), targetFileSystem, targetDir, .getPathFromYarnURL(localResources.get(srcPath.getName() + "/1").getResource()) .getParent(); targetFileSystem.listFiles(workDir, true); HashMap<String /* (relative) path */, /* contents */ String> targetFiles = new HashMap<>(4); try (FSDataInputStream in = targetFileSystem.open(targetFile.getPath())) { String absolutePathString = targetFile.getPath().toString(); String relativePath = absolutePathString.substring(workDirPrefixLength);
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
/** * Creates a YARN resource for the remote object at the given location. * * @param remoteRsrcPath remote location of the resource * @param resourceSize size of the resource * @param resourceModificationTime last modification time of the resource * * @return YARN resource */ private static LocalResource registerLocalResource( Path remoteRsrcPath, long resourceSize, long resourceModificationTime) { LocalResource localResource = Records.newRecord(LocalResource.class); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(resourceSize); localResource.setTimestamp(resourceModificationTime); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
private Path copy(Path sCopy, Path dstdir) throws IOException { FileSystem sourceFs = sCopy.getFileSystem(conf); Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName()); FileStatus sStat = sourceFs.getFileStatus(sCopy); if (sStat.getModificationTime() != resource.getTimestamp()) { throw new IOException("Resource " + sCopy + " changed on src filesystem (expected " + resource.getTimestamp() + ", was " + sStat.getModificationTime()); } if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) { if (!isPublic(sourceFs, sCopy, sStat, statCache)) { throw new IOException("Resource " + sCopy + " is not publicly accessable and as such cannot be part of the" + " public cache."); } } FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false, true, conf); return dCopy; }
Path renamedScriptPath; if (Shell.WINDOWS) { renamedScriptPath = new Path(jstormMasterContext.scriptPath + ".bat"); } else { renamedScriptPath = new Path(jstormMasterContext.scriptPath + ".sh"); yarnUrl = ConverterUtils.getYarnUrlFromURI( new URI(renamedScriptPath.toString())); } catch (URISyntaxException e) { LOG.error("Error when trying to use shell script path specified" jarUrl = ConverterUtils.getYarnUrlFromURI( new URI(jstormMasterContext.appMasterJarPath.toString())); } catch (URISyntaxException e) { LocalResource shellRsrc = LocalResource.newInstance(yarnUrl, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, jstormMasterContext.shellScriptPathLen, jstormMasterContext.shellScriptPathTimestamp); localResources.put(JOYConstants.ExecShellStringPath, shellRsrc); LocalResource jarRsrc = LocalResource.newInstance(jarUrl, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, jstormMasterContext.jarPathLen, jstormMasterContext.jarTimestamp); localResources.put(JOYConstants.appMasterJarPath, jarRsrc); LOG.info(shellRsrc.getResource().getFile()); LOG.info(jarRsrc.getResource().getFile());
private List<LocalResource> getDataNodeBlockListingFiles() throws IOException { Path blockListDirPath = new Path(System.getenv().get(DynoConstants.BLOCK_LIST_PATH_ENV)); LOG.info("Looking for block listing files in " + blockListDirPath); FileSystem blockZipFS = blockListDirPath.getFileSystem(conf); List<LocalResource> files = new LinkedList<>(); for (FileStatus stat : blockZipFS.listStatus(blockListDirPath, DynoConstants.BLOCK_LIST_FILE_FILTER)) { LocalResource blockListResource = LocalResource.newInstance( ConverterUtils.getYarnUrlFromPath(stat.getPath()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, stat.getLen(), stat.getModificationTime()); files.add(blockListResource); } return files; }
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String fileName, Map<String, LocalResource> localResources) throws IOException { Path dstPath = new Path(fileDstPath, fileName); dstPath = fs.makeQualified(dstPath); fs.copyFromLocalFile(false, true, new Path(fileSrcPath), dstPath); FileStatus fileStatus = fs.getFileStatus(dstPath); LocalResource localRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dstPath.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime()); localResources.put(fileName, localRsrc); } }
private LocalResource toLocalResource(Path path, LocalResourceVisibility visibility) throws IOException { FileSystem fs = path.getFileSystem(clusterConf.conf()); FileStatus stat = fs.getFileStatus(path); return LocalResource.newInstance( ConverterUtils.getYarnUrlFromPath(path), LocalResourceType.FILE, visibility, stat.getLen(), stat.getModificationTime() ); }
/** * Adds resource to HDFS and local resources map. * @param fs HDFS file system reference * @param srcPath Source path of resource. If no scheme is included, assumed to be on local filesystem. * @param resourceType the type of the src file * @param dstPath name of the resource after localization * @param localResources the local resources map * @throws IOException error when writing to HDFS */ private void addResource(FileSystem fs, String srcPath, LocalResourceType resourceType, String dstPath, Map<String, LocalResource> localResources) throws IOException { Path src = new Path(srcPath); Path dst = new Path(appResourcesPath, dstPath); HdfsUtils.copySrcToDest(src, dst, hdfsConf); fs.setPermission(dst, new FsPermission((short) 0770)); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), resourceType, LocalResourceVisibility.PRIVATE, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(dstPath, scRsrc); }
final Path jobJarPath = new Path(jobJar); final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf); Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(), jobJarFs.getWorkingDirectory()); LocalResourceVisibility jobJarViz = conf.getBoolean(MRJobConfig.JOBJAR_VISIBILITY, String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); rc.setPattern(pattern); localResources.put(MRJobConfig.JOB_JAR, rc); LOG.info("The job-jar file on the remote FS is "
static LocalResource createJar(FileContext files, Path p, LocalResourceVisibility vis) throws IOException { LOG.info("Create jar file " + p); File jarFile = new File((files.makeQualified(p)).toUri()); FileOutputStream stream = new FileOutputStream(jarFile); LOG.info("Create jar out stream "); JarOutputStream out = new JarOutputStream(stream, new Manifest()); LOG.info("Done writing jar stream "); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(p)); FileStatus status = files.getFileStatus(p); ret.setSize(status.getLen()); ret.setTimestamp(status.getModificationTime()); ret.setType(LocalResourceType.PATTERN); ret.setVisibility(vis); ret.setPattern("classes/.*"); return ret; }
public static void addEnvironmentForResource(LocalResource resource, FileSystem fs, String envPrefix, Map<String, String> env) throws IOException { Path resourcePath = new Path(fs.getHomeDirectory(), resource.getResource().getFile()); FileStatus resourceStatus = fs.getFileStatus(resourcePath); long resourceLength = resourceStatus.getLen(); long resourceTimestamp = resourceStatus.getModificationTime(); env.put(envPrefix + Constants.PATH_SUFFIX, resourcePath.toString()); env.put(envPrefix + Constants.LENGTH_SUFFIX, Long.toString(resourceLength)); env.put(envPrefix + Constants.TIMESTAMP_SUFFIX, Long.toString(resourceTimestamp)); }
/** * Create a {@link LocalResource} record with all the given parameters. */ private static LocalResource createLocalResource(FileSystem fc, Path file, LocalResourceType type, LocalResourceVisibility visibility) throws IOException { FileStatus fstat = fc.getFileStatus(file); URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat .getPath())); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); }
private Map.Entry<String, LocalResource> localizeResource(FileStatus status) { URL url = ConverterUtils.getYarnUrlFromURI( status.getPath().toUri()); LocalResource resource = LocalResource.newInstance(url, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, status.getLen(), status.getModificationTime()); String name = status.getPath().getName(); return new AbstractMap.SimpleEntry<>(name, resource); }
public void testCreateLocalResourceForConfigurationFile() throws Exception { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); String filename = "foo.xml"; Configuration conf = has.createConfiguration(getNameNodeUri()); conf.set("foo", "bar"); LocalResource lRes = has.createLocalResourceForConfigurationFile(filename, getTestUser(), conf, getFileSystem().getUri(), getFsTestCaseDir()); assertNotNull(lRes); assertEquals(LocalResourceType.FILE, lRes.getType()); assertEquals(LocalResourceVisibility.APPLICATION, lRes.getVisibility()); Path resPath = ConverterUtils.getPathFromYarnURL(lRes.getResource()); assertEquals(new Path(getFsTestCaseDir(), "foo.xml"), resPath); Configuration conf2 = new Configuration(false); conf2.addResource(getFileSystem().open(resPath)); assertEquals("bar", conf2.get("foo")); } }
private static LocalResource createResource(String url) { LocalResource res = mock(LocalResource.class); when(res.getResource()).thenReturn(URL.fromPath(new Path(url))); return res; }
Map<String, LocalResource> localResources = new HashMap<>(); Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE); defaultFileContext.makeQualified(new Path(jobSubmitDir)))); LOG.debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir); jobConfPath, LocalResourceType.FILE)); if (jobConf.get(MRJobConfig.JAR) != null) { Path jobJarPath = new Path(jobConf.get(MRJobConfig.JAR)); String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); rc.setPattern(pattern); localResources.put(MRJobConfig.JOB_JAR, rc); } else {
URL destUrl; try { destUrl = ConverterUtils.getYarnUrlFromPath( FileContext.getFileContext().makeQualified(dfsPath)); } catch (UnsupportedFileSystemException e) { throw new DfsFacadeException( "Unable to convert dfs file to a URL: " + dfsPath.toString(), e); LocalResource resource = LocalResource.newInstance(destUrl, type, visibility, dfsFileStatus.getLen(), dfsFileStatus.getModificationTime()); return resource;