private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); }
/** * Creates a YARN resource for the remote object at the given location. * * @param remoteRsrcPath remote location of the resource * @param resourceSize size of the resource * @param resourceModificationTime last modification time of the resource * * @return YARN resource */ private static LocalResource registerLocalResource( Path remoteRsrcPath, long resourceSize, long resourceModificationTime) { LocalResource localResource = Records.newRecord(LocalResource.class); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(resourceSize); localResource.setTimestamp(resourceModificationTime); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
yarnUrl = ConverterUtils.getYarnUrlFromURI( new URI(renamedScriptPath.toString())); } catch (URISyntaxException e) { jarUrl = ConverterUtils.getYarnUrlFromURI( new URI(jstormMasterContext.appMasterJarPath.toString())); } catch (URISyntaxException e) {
private Map.Entry<String, LocalResource> localizeResource(FileStatus status) { URL url = ConverterUtils.getYarnUrlFromURI( status.getPath().toUri()); LocalResource resource = LocalResource.newInstance(url, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, status.getLen(), status.getModificationTime()); String name = status.getPath().getName(); return new AbstractMap.SimpleEntry<>(name, resource); }
private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = appName + "/" + appId + "/" + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem .create(fs, dst, new FsPermission((short) 0710)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } fs.setPermission(dst, new FsPermission((short)0755)); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); return dst; }
public static URL getYarnUrlFromPath(Path path) { return getYarnUrlFromURI(path.toUri()); }
public static URL getYarnUrlFromPath(Path path) { return getYarnUrlFromURI(path.toUri()); }
public static LocalResource newLocalResource(URI uri, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, boolean shouldBeUploadedToSharedCache) { return newLocalResource(ConverterUtils.getYarnUrlFromURI(uri), type, visibility, size, timestamp, shouldBeUploadedToSharedCache); }
public static LocalResource newLocalResource(URI uri, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, boolean shouldBeUploadedToSharedCache) { return newLocalResource(ConverterUtils.getYarnUrlFromURI(uri), type, visibility, size, timestamp, shouldBeUploadedToSharedCache); }
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String fileName, Map<String, LocalResource> localResources) throws IOException { Path dstPath = new Path(fileDstPath, fileName); dstPath = fs.makeQualified(dstPath); fs.copyFromLocalFile(false, true, new Path(fileSrcPath), dstPath); FileStatus fileStatus = fs.getFileStatus(dstPath); LocalResource localRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dstPath.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime()); localResources.put(fileName, localRsrc); } }
/** * Adds resource to HDFS and local resources map. * @param fs HDFS file system reference * @param srcPath Source path of resource. If no scheme is included, assumed to be on local filesystem. * @param resourceType the type of the src file * @param dstPath name of the resource after localization * @param localResources the local resources map * @throws IOException error when writing to HDFS */ private void addResource(FileSystem fs, String srcPath, LocalResourceType resourceType, String dstPath, Map<String, LocalResource> localResources) throws IOException { Path src = new Path(srcPath); Path dst = new Path(appResourcesPath, dstPath); HdfsUtils.copySrcToDest(src, dst, hdfsConf); fs.setPermission(dst, new FsPermission((short) 0770)); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), resourceType, LocalResourceVisibility.PRIVATE, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(dstPath, scRsrc); }
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
/** * Boilerplate to add a file to the local resources.. * * @param localResources * the LocalResources map to populate. * @param fs * handle to the HDFS file system. * @param target * the file to send to the remote container. */ public static void addFileToResourceMap(Map<String, LocalResource> localResources, FileSystem fs, Path target) throws IOException { LocalResource resource = Records.newRecord(LocalResource.class); FileStatus destStatus = fs.getFileStatus(target); resource.setResource(ConverterUtils.getYarnUrlFromURI(target.toUri())); resource.setSize(destStatus.getLen()); resource.setTimestamp(destStatus.getModificationTime()); resource.setType(LocalResourceType.FILE); // use FILE, even for jars! resource.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put(target.getName(), resource); LOG.info("Registered file in LocalResources :{} ", target); }
/** * Boilerplate to add a file to the local resources.. * * @param localResources * the LocalResources map to populate. * @param fs * handle to the HDFS file system. * @param target * the file to send to the remote container. */ public static void addFileToResourceMap(Map<String, LocalResource> localResources, FileSystem fs, Path target) throws IOException { LocalResource resource = Records.newRecord(LocalResource.class); FileStatus destStatus = fs.getFileStatus(target); resource.setResource(ConverterUtils.getYarnUrlFromURI(target.toUri())); resource.setSize(destStatus.getLen()); resource.setTimestamp(destStatus.getModificationTime()); resource.setType(LocalResourceType.FILE); // use FILE, even for jars! resource.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put(target.getName(), resource); LOG.info("Registered file in LocalResources :{} ", target); }
/** * Creates a YARN resource for the remote object at the given location. * * @param remoteRsrcPath remote location of the resource * @param resourceSize size of the resource * @param resourceModificationTime last modification time of the resource * * @return YARN resource */ private static LocalResource registerLocalResource( Path remoteRsrcPath, long resourceSize, long resourceModificationTime) { LocalResource localResource = Records.newRecord(LocalResource.class); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(resourceSize); localResource.setTimestamp(resourceModificationTime); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
public static YarnLocalResource createLocalResource(LocalFile localFile) { Preconditions.checkArgument(localFile.getLastModified() >= 0, "Last modified time should be >= 0."); Preconditions.checkArgument(localFile.getSize() >= 0, "File size should be >= 0."); YarnLocalResource resource = createAdapter(YarnLocalResource.class); resource.setVisibility(LocalResourceVisibility.APPLICATION); resource.setResource(ConverterUtils.getYarnUrlFromURI(localFile.getURI())); resource.setTimestamp(localFile.getLastModified()); resource.setSize(localFile.getSize()); return setLocalResourceType(resource, localFile); }
/** * Creates a YARN resource for the remote object at the given location. * * @param remoteRsrcPath remote location of the resource * @param resourceSize size of the resource * @param resourceModificationTime last modification time of the resource * * @return YARN resource */ private static LocalResource registerLocalResource( Path remoteRsrcPath, long resourceSize, long resourceModificationTime) { LocalResource localResource = Records.newRecord(LocalResource.class); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(resourceSize); localResource.setTimestamp(resourceModificationTime); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
public static YarnLocalResource createLocalResource(LocalFile localFile) { Preconditions.checkArgument(localFile.getLastModified() >= 0, "Last modified time should be >= 0."); Preconditions.checkArgument(localFile.getSize() >= 0, "File size should be >= 0."); YarnLocalResource resource = createAdapter(YarnLocalResource.class); resource.setVisibility(LocalResourceVisibility.APPLICATION); resource.setResource(ConverterUtils.getYarnUrlFromURI(localFile.getURI())); resource.setTimestamp(localFile.getLastModified()); resource.setSize(localFile.getSize()); return setLocalResourceType(resource, localFile); }