out.println(resource.getSize()); out.print(" Timestamp: "); out.println(DoYUtil.toIsoTime(resource.getTimestamp())); out.print(" Type: "); out.println(resource.getType().toString());
@Override public long getTimestamp() { return localResource.getTimestamp(); }
@Override public long getTimestamp() { return localResource.getTimestamp(); }
@Override public long getTimestamp() { return localResource.getTimestamp(); }
private Path copy(Path sCopy, Path dstdir) throws IOException { FileSystem sourceFs = sCopy.getFileSystem(conf); Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName()); FileStatus sStat = sourceFs.getFileStatus(sCopy); if (sStat.getModificationTime() != resource.getTimestamp()) { throw new IOException("Resource " + sCopy + " changed on src filesystem (expected " + resource.getTimestamp() + ", was " + sStat.getModificationTime()); } if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) { if (!isPublic(sourceFs, sCopy, sStat, statCache)) { throw new IOException("Resource " + sCopy + " is not publicly accessable and as such cannot be part of the" + " public cache."); } } FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false, true, conf); return dCopy; }
private Path copy(Path sCopy, Path dstdir) throws IOException { FileSystem sourceFs = sCopy.getFileSystem(conf); Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName()); FileStatus sStat = sourceFs.getFileStatus(sCopy); if (sStat.getModificationTime() != resource.getTimestamp()) { throw new IOException("Resource " + sCopy + " changed on src filesystem (expected " + resource.getTimestamp() + ", was " + sStat.getModificationTime()); } if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) { if (!isPublic(sourceFs, sCopy, sStat, statCache)) { throw new IOException("Resource " + sCopy + " is not publicly accessable and as such cannot be part of the" + " public cache."); } } FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false, true, conf); return dCopy; }
/** * Creates the JAR file with the global files on the driver and then uploads it to the job folder on * (H)DFS. * * @return the map to be used as the "global" resources when submitting Evaluators. * @throws IOException if the creation of the JAR or the upload fails */ @Override public synchronized Map<String, LocalResource> call() throws IOException { final Map<String, LocalResource> globalResources = new HashMap<>(1); if (!this.isUploaded){ this.pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar()); this.isUploaded = true; } final LocalResource updatedGlobalJarResource = this.uploader.makeLocalResourceForJarFile(this.pathToGlobalJar); if (this.globalJarResource != null && this.globalJarResource.getTimestamp() != updatedGlobalJarResource.getTimestamp()) { LOG.log(Level.WARNING, "The global JAR LocalResource timestamp has been changed from " + this.globalJarResource.getTimestamp() + " to " + updatedGlobalJarResource.getTimestamp()); } this.globalJarResource = updatedGlobalJarResource; // For now, always rewrite the information due to REEF-348 globalResources.put(this.fileNames.getGlobalFolderPath(), updatedGlobalJarResource); return globalResources; }
/** * Localize files. * @param destination destination directory * @throws IOException cannot read or write file * @throws YarnException subcommand returned an error */ private void verifyAndCopy(Path destination) throws IOException, YarnException { final Path sCopy; try { sCopy = resource.getResource().toPath(); } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); } FileSystem sourceFs = sCopy.getFileSystem(conf); FileStatus sStat = sourceFs.getFileStatus(sCopy); if (sStat.getModificationTime() != resource.getTimestamp()) { throw new IOException("Resource " + sCopy + " changed on src filesystem (expected " + resource.getTimestamp() + ", was " + sStat.getModificationTime()); } if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) { if (!isPublic(sourceFs, sCopy, sStat, statCache)) { throw new IOException("Resource " + sCopy + " is not publicly accessible and as such cannot be part of the" + " public cache."); } } downloadAndUnpack(sCopy, destination); }
FileStatus status = remoteFs.getFileStatus(remotePath); if (status.getModificationTime() != resource.getTimestamp()) { LOG.warn("The remote file " + remotePath + " has changed since it's localized; will not consider it for upload");
@Private public static TezLocalResource convertYarnLocalResourceToTez(LocalResource lr) throws URISyntaxException { return new TezLocalResource(getURIFromYarnURL(lr.getResource()), lr.getSize(), lr.getTimestamp()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(resource.getResource().toPath(), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(ConverterUtils.getPathFromYarnURL(resource.getResource()), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(ConverterUtils.getPathFromYarnURL(resource.getResource()), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(resource.getResource().toPath(), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
public static List<PlanLocalResource> convertToDAGPlan(Map<String, LocalResource> lrs) { List<PlanLocalResource> planLrs = Lists.newArrayListWithCapacity(lrs.size()); for (Entry<String, LocalResource> entry : lrs.entrySet()) { PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder(); String key = entry.getKey(); LocalResource lr = entry.getValue(); localResourcesBuilder.setName(key); localResourcesBuilder.setUri( DagTypeConverters.convertToDAGPlan(lr.getResource())); localResourcesBuilder.setSize(lr.getSize()); localResourcesBuilder.setTimeStamp(lr.getTimestamp()); localResourcesBuilder.setType( DagTypeConverters.convertToDAGPlan(lr.getType())); localResourcesBuilder.setVisibility( DagTypeConverters.convertToDAGPlan(lr.getVisibility())); if (lr.getType() == LocalResourceType.PATTERN) { if (lr.getPattern() == null || lr.getPattern().isEmpty()) { throw new TezUncheckedException("LocalResource type set to pattern" + " but pattern is null or empty"); } localResourcesBuilder.setPattern(lr.getPattern()); } planLrs.add(localResourcesBuilder.build()); } return planLrs; }
public static PlanLocalResource convertLocalResourceToPlanLocalResource( String name, LocalResource lr) { PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder(); localResourcesBuilder.setName(name); localResourcesBuilder.setUri( DagTypeConverters.convertToDAGPlan(lr.getResource())); localResourcesBuilder.setSize(lr.getSize()); localResourcesBuilder.setTimeStamp(lr.getTimestamp()); localResourcesBuilder.setType( DagTypeConverters.convertToDAGPlan(lr.getType())); localResourcesBuilder.setVisibility( DagTypeConverters.convertToDAGPlan(lr.getVisibility())); if (lr.getType() == LocalResourceType.PATTERN) { if (lr.getPattern() == null || lr.getPattern().isEmpty()) { throw new TezUncheckedException("LocalResource type set to pattern" + " but pattern is null or empty"); } localResourcesBuilder.setPattern(lr.getPattern()); } return localResourcesBuilder.build(); }