/** * Based on the given {@link TwillSpecification}, upload LocalFiles to Yarn Cluster. * @param twillSpec The {@link TwillSpecification} for populating resource. */ private Multimap<String, LocalFile> populateRunnableLocalFiles(TwillSpecification twillSpec) throws IOException { Multimap<String, LocalFile> localFiles = HashMultimap.create(); LOG.debug("Populating Runnable LocalFiles"); for (Map.Entry<String, RuntimeSpecification> entry: twillSpec.getRunnables().entrySet()) { String runnableName = entry.getKey(); for (LocalFile localFile : entry.getValue().getLocalFiles()) { Location location; URI uri = localFile.getURI(); if (appLocation.toURI().getScheme().equals(uri.getScheme())) { // If the source file location is having the same scheme as the target location, no need to copy location = appLocation.getLocationFactory().create(uri); } else { URL url = uri.toURL(); LOG.debug("Create and copy {} : {}", runnableName, url); // Preserves original suffix for expansion. location = copyFromURL(url, createTempLocation(Paths.addExtension(url.getFile(), localFile.getName()))); LOG.debug("Done {} : {}", runnableName, url); } localFiles.put(runnableName, new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern())); } } LOG.debug("Done Runnable LocalFiles"); return localFiles; }
/** * Based on the given {@link TwillSpecification}, upload LocalFiles to Yarn Cluster. * @param twillSpec The {@link TwillSpecification} for populating resource. */ private Multimap<String, LocalFile> populateRunnableLocalFiles(TwillSpecification twillSpec) throws IOException { Multimap<String, LocalFile> localFiles = HashMultimap.create(); LOG.debug("Populating Runnable LocalFiles"); for (Map.Entry<String, RuntimeSpecification> entry: twillSpec.getRunnables().entrySet()) { String runnableName = entry.getKey(); for (LocalFile localFile : entry.getValue().getLocalFiles()) { Location location; URI uri = localFile.getURI(); if (appLocation.toURI().getScheme().equals(uri.getScheme())) { // If the source file location is having the same scheme as the target location, no need to copy location = appLocation.getLocationFactory().create(uri); } else { URL url = uri.toURL(); LOG.debug("Create and copy {} : {}", runnableName, url); // Preserves original suffix for expansion. location = copyFromURL(url, createTempLocation(Paths.addExtension(url.getFile(), localFile.getName()))); LOG.debug("Done {} : {}", runnableName, url); } localFiles.put(runnableName, new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern())); } } LOG.debug("Done Runnable LocalFiles"); return localFiles; }