@Override public void setPattern(String pattern) { localResource.setPattern(pattern); } }
@Override public void setPattern(String pattern) { localResource.setPattern(pattern); } }
@Override public void setPattern(String pattern) { localResource.setPattern(pattern); } }
String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); rc.setPattern(pattern); localResources.put(MRJobConfig.JOB_JAR, rc); LOG.info("The job-jar file on the remote FS is "
String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); rc.setPattern(pattern); localResources.put(MRJobConfig.JOB_JAR, rc); } else {
resource.setPattern( pattern );
resource.setPattern( pattern );
static LocalResource createJar(FileContext files, Path p, LocalResourceVisibility vis) throws IOException { LOG.info("Create jar file " + p); File jarFile = new File((files.makeQualified(p)).toUri()); FileOutputStream stream = new FileOutputStream(jarFile); LOG.info("Create jar out stream "); JarOutputStream out = new JarOutputStream(stream, new Manifest()); LOG.info("Done writing jar stream "); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(p)); FileStatus status = files.getFileStatus(p); ret.setSize(status.getLen()); ret.setTimestamp(status.getModificationTime()); ret.setType(LocalResourceType.PATTERN); ret.setVisibility(vis); ret.setPattern("classes/.*"); return ret; }
ret.setType(LocalResourceType.PATTERN); ret.setVisibility(vis); ret.setPattern("classes/.*"); return ret;
static LocalResource createJar(FileContext files, Path p, LocalResourceVisibility vis) throws IOException { LOG.info("Create jar file " + p); File jarFile = new File((files.makeQualified(p)).toUri()); FileOutputStream stream = new FileOutputStream(jarFile); LOG.info("Create jar out stream "); JarOutputStream out = new JarOutputStream(stream, new Manifest()); LOG.info("Done writing jar stream "); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(URL.fromPath(p)); FileStatus status = files.getFileStatus(p); ret.setSize(status.getLen()); ret.setTimestamp(status.getModificationTime()); ret.setType(LocalResourceType.PATTERN); ret.setVisibility(vis); ret.setPattern("classes/.*"); return ret; }
String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); rc.setPattern(pattern); localResources.put(MRJobConfig.JOB_JAR, rc); LOG.info("The job-jar file on the remote FS is "
next.setType(nextRsrc.getType()); next.setVisibility(evt.getVisibility()); next.setPattern(evt.getPattern()); scheduled.put(nextRsrc, evt); return next;
next.setType(nextRsrc.getType()); next.setVisibility(evt.getVisibility()); next.setPattern(evt.getPattern()); scheduled.put(nextRsrc, evt); return next;
next.setType(nextRsrc.getType()); next.setVisibility(evt.getVisibility()); next.setPattern(evt.getPattern()); scheduled.put(nextRsrc, evt); return next;
next.setType(nextRsrc.getType()); next.setVisibility(evt.getVisibility()); next.setPattern(evt.getPattern()); scheduled.put(nextRsrc, evt); return next;
@Public @Unstable public static LocalResource newInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, String pattern, boolean shouldBeUploadedToSharedCache) { LocalResource resource = Records.newRecord(LocalResource.class); resource.setResource(url); resource.setType(type); resource.setVisibility(visibility); resource.setSize(size); resource.setTimestamp(timestamp); resource.setPattern(pattern); resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return resource; }
@Public @Unstable public static LocalResource newInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, String pattern, boolean shouldBeUploadedToSharedCache) { LocalResource resource = Records.newRecord(LocalResource.class); resource.setResource(url); resource.setType(type); resource.setVisibility(visibility); resource.setSize(size); resource.setTimestamp(timestamp); resource.setPattern(pattern); resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return resource; }
@Public @Unstable public static LocalResource newInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, String pattern, boolean shouldBeUploadedToSharedCache) { LocalResource resource = Records.newRecord(LocalResource.class); resource.setResource(url); resource.setType(type); resource.setVisibility(visibility); resource.setSize(size); resource.setTimestamp(timestamp); resource.setPattern(pattern); resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return resource; }
@Public @Unstable public static LocalResource newInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, String pattern, boolean shouldBeUploadedToSharedCache) { LocalResource resource = Records.newRecord(LocalResource.class); resource.setResource(url); resource.setType(type); resource.setVisibility(visibility); resource.setSize(size); resource.setTimestamp(timestamp); resource.setPattern(pattern); resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return resource; }
public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan( List<PlanLocalResource> localResourcesList) { Map<String, LocalResource> map = new HashMap<String, LocalResource>(); for(PlanLocalResource res : localResourcesList){ LocalResource r = new LocalResourcePBImpl(); //NOTE: have to check every optional field in protobuf generated classes for existence before accessing //else we will receive a default value back, eg "" if(res.hasPattern()){ r.setPattern(res.getPattern()); } r.setResource(convertToYarnURL(res.getUri())); r.setSize(res.getSize()); r.setTimestamp(res.getTimeStamp()); r.setType(DagTypeConverters.convertFromDAGPlan(res.getType())); r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility())); map.put(res.getName(), r); } return map; }