public static Map<String, LocalResource> getResourcesUpdatableForAm( Collection<LocalResource> allNonAppResources) { HashMap<String, LocalResource> allNonAppFileResources = new HashMap<>(); if (allNonAppResources == null) { return allNonAppFileResources; } for (LocalResource lr : allNonAppResources) { if (lr.getType() == LocalResourceType.FILE) { // TEZ AM will only localize FILE (no script operators in the AM) allNonAppFileResources.put(DagUtils.getBaseName(lr), lr); } } return allNonAppFileResources; } }
out.println(DoYUtil.toIsoTime(resource.getTimestamp())); out.print(" Type: "); out.println(resource.getType().toString()); out.print(" Visiblity: "); out.println(resource.getVisibility().toString());
if (additionalLr != null) { for (LocalResource lr: additionalLr) { if (lr.getType() == LocalResourceType.FILE) {
@Override public LocalResourceType getType() { return localResource.getType(); }
@Override public LocalResourceType getType() { return localResource.getType(); }
@Override public LocalResourceType getType() { return localResource.getType(); }
private void checkLocalResources(Map<String, LocalResource> localResources) { for (Map.Entry<String, LocalResource> rsrcEntry : localResources .entrySet()) { if (rsrcEntry.getValue() == null || rsrcEntry.getValue().getResource() == null) { throw new NullPointerException( "Null resource URL for local resource " + rsrcEntry.getKey() + " : " + rsrcEntry.getValue()); } else if (rsrcEntry.getValue().getType() == null) { throw new NullPointerException( "Null resource type for local resource " + rsrcEntry.getKey() + " : " + rsrcEntry.getValue()); } else if (rsrcEntry.getValue().getVisibility() == null) { throw new NullPointerException( "Null resource visibility for local resource " + rsrcEntry.getKey() + " : " + rsrcEntry.getValue()); } } }
static long getEstimatedSize(LocalResource rsrc) { if (rsrc.getSize() < 0) { return -1; } switch (rsrc.getType()) { case ARCHIVE: case PATTERN: return 5 * rsrc.getSize(); case FILE: default: return rsrc.getSize(); } }
static long getEstimatedSize(LocalResource rsrc) { if (rsrc.getSize() < 0) { return -1; } switch (rsrc.getType()) { case ARCHIVE: case PATTERN: return 5 * rsrc.getSize(); case FILE: default: return rsrc.getSize(); } }
static long getEstimatedSize(LocalResource rsrc) { if (rsrc.getSize() < 0) { return -1; } switch (rsrc.getType()) { case ARCHIVE: case PATTERN: return 5 * rsrc.getSize(); case FILE: default: return rsrc.getSize(); } }
static long getEstimatedSize(LocalResource rsrc) { if (rsrc.getSize() < 0) { return -1; } switch (rsrc.getType()) { case ARCHIVE: case PATTERN: return 5 * rsrc.getSize(); case FILE: default: return rsrc.getSize(); } }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(ConverterUtils.getPathFromYarnURL(resource.getResource()), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(ConverterUtils.getPathFromYarnURL(resource.getResource()), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(resource.getResource().toPath(), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(resource.getResource().toPath(), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
public void testCreateLocalResourceForConfigurationFile() throws Exception { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); String filename = "foo.xml"; Configuration conf = has.createConfiguration(getNameNodeUri()); conf.set("foo", "bar"); LocalResource lRes = has.createLocalResourceForConfigurationFile(filename, getTestUser(), conf, getFileSystem().getUri(), getFsTestCaseDir()); assertNotNull(lRes); assertEquals(LocalResourceType.FILE, lRes.getType()); assertEquals(LocalResourceVisibility.APPLICATION, lRes.getVisibility()); Path resPath = ConverterUtils.getPathFromYarnURL(lRes.getResource()); assertEquals(new Path(getFsTestCaseDir(), "foo.xml"), resPath); Configuration conf2 = new Configuration(false); conf2.addResource(getFileSystem().open(resPath)); assertEquals("bar", conf2.get("foo")); } }