private void logResources(List<LocalResource> additionalLr) { // log which resources we're adding (apart from the hive exec) if (!LOG.isDebugEnabled()) return; if (additionalLr == null || additionalLr.size() == 0) { LOG.debug("No local resources to process (other than hive-exec)"); } else { for (LocalResource lr: additionalLr) { LOG.debug("Adding local resource: " + lr.getResource()); } } }
private void logResources(List<LocalResource> additionalLr) { // log which resources we're adding (apart from the hive exec) if (!LOG.isDebugEnabled()) return; if (additionalLr == null || additionalLr.size() == 0) { LOG.debug("No local resources to process (other than hive-exec)"); } else { for (LocalResource lr: additionalLr) { LOG.debug("Adding local resource: " + lr.getResource()); } } }
public static String getBaseName(LocalResource lr) { return FilenameUtils.getName(lr.getResource().getFile()); }
public String getBaseName(LocalResource lr) { return FilenameUtils.getName(lr.getResource().getFile()); }
private static LocalResource createResource(String url) { LocalResource res = mock(LocalResource.class); when(res.getResource()).thenReturn(URL.fromPath(new Path(url))); return res; }
LocalResource resource = resources.get(key); out.print(" URL: "); out.println(resource.getResource().toString()); out.print(" Size: "); out.println(resource.getSize());
localResources.put(JOYConstants.appMasterJarPath, jarRsrc); LOG.info(shellRsrc.getResource().getFile()); LOG.info(jarRsrc.getResource().getFile());
ContainerLaunchContext ctx = Utils.createTaskExecutorContext(flinkConf, yarnConf, env, tmParams, taskManagerConf, workingDirectory, taskManagerMainClass, LOG); assertEquals("file", ctx.getLocalResources().get("flink.jar").getResource().getScheme());
.getPathFromYarnURL(localResources.get(srcPath.getName() + "/1").getResource()) .getParent();
String modelScript = localizeResources(localResources, new Path(request.getPath()), appJarLocation); for(Map.Entry<String, LocalResource> entry : localResources.entrySet()) { LOG.info(entry.getKey() + " localized: " + entry.getValue().getResource() );
@Override public URL getResource() { return localResource.getResource(); }
public static void addEnvironmentForResource(LocalResource resource, FileSystem fs, String envPrefix, Map<String, String> env) throws IOException { Path resourcePath = new Path(fs.getHomeDirectory(), resource.getResource().getFile()); FileStatus resourceStatus = fs.getFileStatus(resourcePath); long resourceLength = resourceStatus.getLen(); long resourceTimestamp = resourceStatus.getModificationTime(); env.put(envPrefix + Constants.PATH_SUFFIX, resourcePath.toString()); env.put(envPrefix + Constants.LENGTH_SUFFIX, Long.toString(resourceLength)); env.put(envPrefix + Constants.TIMESTAMP_SUFFIX, Long.toString(resourceTimestamp)); }
private static URI getLocalResourceUri(LocalResource input) { try { return TezConverterUtils.getURIFromYarnURL(input.getResource()); } catch (URISyntaxException e) { throw new TezUncheckedException("Failed while handling : " + input, e); } }
@Private public static TezLocalResource convertYarnLocalResourceToTez(LocalResource lr) throws URISyntaxException { return new TezLocalResource(getURIFromYarnURL(lr.getResource()), lr.getSize(), lr.getTimestamp()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(ConverterUtils.getPathFromYarnURL(resource.getResource()), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
/** * Wrap API resource to match against cache of localized resources. * @param resource Resource requested by container * @throws URISyntaxException If the path is malformed */ public LocalResourceRequest(LocalResource resource) throws URISyntaxException { this(resource.getResource().toPath(), resource.getTimestamp(), resource.getType(), resource.getVisibility(), resource.getPattern()); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }
private LocalResourceProto buildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr instanceof LocalResourcePBImpl)) { lr = LocalResource.newInstance(lr.getResource(), lr.getType(), lr.getVisibility(), lr.getSize(), lr.getTimestamp(), lr.getPattern()); } lrpb = (LocalResourcePBImpl) lr; return lrpb.getProto(); }