public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(path); localResource.setResource(ConverterUtils.getYarnUrlFromPath(path)); localResource.setSize(fileStatus.getLen()); localResource.setTimestamp(fileStatus.getModificationTime()); localResource.setType(type); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
private LocalResource createLocalResource(FileSystem remoteFs, Path file, LocalResourceType type, LocalResourceVisibility visibility) { FileStatus fstat = null; try { fstat = remoteFs.getFileStatus(file); } catch (IOException e) { e.printStackTrace(); } URL resourceURL = ConverterUtils.getYarnUrlFromPath(file); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); LOG.info("Resource modification time: " + resourceModificationTime + " for " + file); LocalResource lr = Records.newRecord(LocalResource.class); lr.setResource(resourceURL); lr.setType(type); lr.setSize(resourceSize); lr.setVisibility(visibility); lr.setTimestamp(resourceModificationTime); return lr; }
/** * @param file Path. * @param fs File system. * @param type Local resource type. * @throws Exception If failed. */ public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type) throws Exception { LocalResource resource = Records.newRecord(LocalResource.class); file = fs.makeQualified(file); FileStatus stat = fs.getFileStatus(file); resource.setResource(ConverterUtils.getYarnUrlFromPath(file)); resource.setSize(stat.getLen()); resource.setTimestamp(stat.getModificationTime()); resource.setType(type); resource.setVisibility(LocalResourceVisibility.APPLICATION); return resource; }
private LocalResource createLocalResource(FileSystem remoteFs, Path file, LocalResourceType type, LocalResourceVisibility visibility) { FileStatus fstat = null; try { fstat = remoteFs.getFileStatus(file); } catch (IOException e) { e.printStackTrace(); } URL resourceURL = ConverterUtils.getYarnUrlFromPath(file); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); LOG.info("Resource modification time: " + resourceModificationTime + " for " + file); LocalResource lr = Records.newRecord(LocalResource.class); lr.setResource(resourceURL); lr.setType(type); lr.setSize(resourceSize); lr.setVisibility(visibility); lr.setTimestamp(resourceModificationTime); return lr; }
/** * Add a file as a Yarn {@link org.apache.hadoop.yarn.api.records.LocalResource}. * * @param fs a {@link FileSystem} instance * @param destFilePath the destination file path * @param resourceType the {@link org.apache.hadoop.yarn.api.records.LocalResourceType} of the file * @param resourceMap a {@link Map} of file names to their corresponding * {@link org.apache.hadoop.yarn.api.records.LocalResource}s * @throws IOException if there's something wrong adding the file as a * {@link org.apache.hadoop.yarn.api.records.LocalResource} */ public static void addFileAsLocalResource(FileSystem fs, Path destFilePath, LocalResourceType resourceType, Map<String, LocalResource> resourceMap) throws IOException { LocalResource fileResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(destFilePath); fileResource.setResource(ConverterUtils.getYarnUrlFromPath(destFilePath)); fileResource.setSize(fileStatus.getLen()); fileResource.setTimestamp(fileStatus.getModificationTime()); fileResource.setType(resourceType); fileResource.setVisibility(LocalResourceVisibility.APPLICATION); resourceMap.put(destFilePath.getName(), fileResource); }
URL destUrl; try { destUrl = ConverterUtils.getYarnUrlFromPath( FileContext.getFileContext().makeQualified(dfsPath)); } catch (UnsupportedFileSystemException e) {
private LocalResource toLocalResource(Path path, LocalResourceVisibility visibility) throws IOException { FileSystem fs = path.getFileSystem(clusterConf.conf()); FileStatus stat = fs.getFileStatus(path); return LocalResource.newInstance( ConverterUtils.getYarnUrlFromPath(path), LocalResourceType.FILE, visibility, stat.getLen(), stat.getModificationTime() ); }
@Override public URL getResource() { return ConverterUtils.getYarnUrlFromPath(loc); }
public static void addFileToLocalResources(final String name, final FileStatus fileStatus, final LocalResourceType type, final Map<String, LocalResource> localResources) { final LocalResource localResource = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fileStatus.getPath()), type, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime()); localResources.put(name, localResource); }
private void configureLocalResourceForPath(LocalResource rsrc, Path path) throws IOException { FileSystem fs = FileSystem.get(conf); FileStatus stat = fs.getFileStatus(path); rsrc.setSize(stat.getLen()); rsrc.setTimestamp(stat.getModificationTime()); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(path)); }
private void setupAppMasterJar(Path jarPath, LocalResource appMasterJar) throws IOException { FileStatus jarStat = FileSystem.get(conf).getFileStatus(jarPath); appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); appMasterJar.setSize(jarStat.getLen()); appMasterJar.setTimestamp(jarStat.getModificationTime()); appMasterJar.setType(LocalResourceType.FILE); appMasterJar.setVisibility(LocalResourceVisibility.PUBLIC); }
static LocalResource newYarnAppResource(FileSystem fs, Path path, LocalResourceType type, LocalResourceVisibility vis) throws IOException { Path qualified = fs.makeQualified(path); FileStatus status = fs.getFileStatus(qualified); LocalResource resource = Records.newRecord(LocalResource.class); resource.setType(type); resource.setVisibility(vis); resource.setResource(ConverterUtils.getYarnUrlFromPath(qualified)); resource.setTimestamp(status.getModificationTime()); resource.setSize(status.getLen()); return resource; }
public static ResourceLocalizationSpec newResourceLocalizationSpec( LocalResource rsrc, Path path) { URL local = ConverterUtils.getYarnUrlFromPath(path); ResourceLocalizationSpec resourceLocalizationSpec = Records.newRecord(ResourceLocalizationSpec.class); resourceLocalizationSpec.setDestinationDirectory(local); resourceLocalizationSpec.setResource(rsrc); return resourceLocalizationSpec; }
public static ResourceLocalizationSpec newResourceLocalizationSpec( LocalResource rsrc, Path path) { URL local = ConverterUtils.getYarnUrlFromPath(path); ResourceLocalizationSpec resourceLocalizationSpec = Records.newRecord(ResourceLocalizationSpec.class); resourceLocalizationSpec.setDestinationDirectory(local); resourceLocalizationSpec.setResource(rsrc); return resourceLocalizationSpec; }
private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type) throws IOException { LocalResource rsrc = Records.newRecord(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs .getDefaultFileSystem().resolvePath(rsrcStat.getPath()))); rsrc.setSize(rsrcStat.getLen()); rsrc.setTimestamp(rsrcStat.getModificationTime()); rsrc.setType(type); rsrc.setVisibility(LocalResourceVisibility.APPLICATION); return rsrc; }
private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type) throws IOException { LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs .getDefaultFileSystem().resolvePath(rsrcStat.getPath()))); rsrc.setSize(rsrcStat.getLen()); rsrc.setTimestamp(rsrcStat.getModificationTime()); rsrc.setType(type); rsrc.setVisibility(LocalResourceVisibility.APPLICATION); return rsrc; }
@Test public void testConvertUrlWithNoPort() throws URISyntaxException { Path expectedPath = new Path("hdfs://foo.com"); URL url = ConverterUtils.getYarnUrlFromPath(expectedPath); Path actualPath = ConverterUtils.getPathFromYarnURL(url); assertEquals(expectedPath, actualPath); }
@Test public void testConvertUrlWithUserinfo() throws URISyntaxException { Path expectedPath = new Path("foo://username:password@example.com:8042"); URL url = ConverterUtils.getYarnUrlFromPath(expectedPath); Path actualPath = ConverterUtils.getPathFromYarnURL(url); assertEquals(expectedPath, actualPath); }
public static LocalResource convertPlanLocalResourceToLocalResource( PlanLocalResource plr) { return LocalResource.newInstance( ConverterUtils.getYarnUrlFromPath(new Path(plr.getUri())), DagTypeConverters.convertFromDAGPlan(plr.getType()), DagTypeConverters.convertFromDAGPlan(plr.getVisibility()), plr.getSize(), plr.getTimeStamp(), plr.hasPattern() ? plr.getPattern() : null); }
static LocalResource createFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException { createFile(files, p, len, r); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(p)); ret.setSize(len); ret.setType(LocalResourceType.FILE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(p).getModificationTime()); return ret; }