public void init() throws IOException { if (localDir.exists()) { // TODO: We don't want some random jars of unknown provenance sitting around. Or do we care? // Ideally, we should try to reuse jars and verify using some checksum. FileUtils.deleteDirectory(localDir); } this.resourceDownloader = new ResourceDownloader(conf, localDir.getAbsolutePath()); workThread.start(); }
private List<URI> resolveAndDownloadInternal(URI source, String subDir, boolean convertToUnix, boolean isLocalAllowed) throws URISyntaxException, IOException { switch (getURLType(source)) { case FILE: return isLocalAllowed ? Collections.singletonList(source) : null; case IVY: return dependencyResolver.downloadDependencies(source); case HDFS: case OTHER: return Collections.singletonList(createURI(downloadResource(source, subDir, convertToUnix))); default: throw new AssertionError(getURLType(source)); } }
public ResourceDownloader(Configuration conf, String resourceDirPath) { this.dependencyResolver = new DependencyResolver(); this.conf = conf; this.resourceDir = new File(resourceDirPath); ensureDirectory(resourceDir); }
public List<URI> resolveAndDownload(String source, boolean convertToUnix) throws URISyntaxException, IOException { return resolveAndDownloadInternal(createURI(source), null, convertToUnix, true); }
public static boolean isIvyUri(String value) throws URISyntaxException { return UriType.IVY == getURLType(createURI(value)); }
new ResourceDownloader(conf, udfDir.toUri().normalize().getPath()); List<Function> fns = hive.getAllFunctions(); Set<URI> srcUris = new HashSet<>(); srcUris.add(ResourceDownloader.createURI(resource.getUri())); List<URI> localUris = resourceDownloader.downloadExternal(srcUri, null, false); for(URI dst : localUris) { LOG.warn("Downloaded " + dst + " from " + srcUri);
public static boolean isFileUri(String value) { String scheme = null; try { scheme = createURI(value).getScheme(); } catch (URISyntaxException ex) { throw new RuntimeException(ex); } return (scheme == null) || scheme.equalsIgnoreCase("file"); }
private String fetchFilesNotInLocalFilesystem(String cmd) { SessionState ss = SessionState.get(); String progName = getScriptProgName(cmd); if (!ResourceDownloader.isFileUri(progName)) { String filePath = ss.add_resource(ResourceType.FILE, progName, true); Path p = new Path(filePath); String fileName = p.getName(); String scriptArgs = getScriptArgs(cmd); String finalCmd = fileName + scriptArgs; return finalCmd; } return cmd; }
@VisibleForTesting protected List<URI> resolveAndDownload(ResourceType resourceType, String value, boolean convertToUnix) throws URISyntaxException, IOException { List<URI> uris = resourceDownloader.resolveAndDownload(value, convertToUnix); if (ResourceDownloader.isHdfsUri(value)) { assert uris.size() == 1 : "There should only be one URI localized-resource."; resourceMaps.getLocalHdfsLocationMap(resourceType).put(uris.get(0).toString(), value); } return uris; }
public List<URI> downloadExternal(URI source, String subDir, boolean convertToUnix) throws URISyntaxException, IOException { return resolveAndDownloadInternal(source, subDir, convertToUnix, false); }
@VisibleForTesting protected List<URI> resolveAndDownload(String value, boolean convertToUnix) throws URISyntaxException, IOException { return resourceDownloader.resolveAndDownload(value, convertToUnix); }
new ResourceDownloader(conf, udfDir.toUri().normalize().getPath()); List<Function> fns = hive.getAllFunctions(); Set<URI> srcUris = new HashSet<>(); srcUris.add(ResourceDownloader.createURI(resource.getUri())); List<URI> localUris = resourceDownloader.downloadExternal(srcUri, null, false); for(URI dst : localUris) { LOG.warn("Downloaded " + dst + " from " + srcUri);
public static boolean isHdfsUri(String value) throws URISyntaxException { return UriType.HDFS == getURLType(createURI(value)); }
public List<URI> resolveAndDownload(String source, boolean convertToUnix) throws URISyntaxException, IOException { return resolveAndDownloadInternal(createURI(source), null, convertToUnix, true); }
public static boolean isIvyUri(String value) throws URISyntaxException { return "ivy".equalsIgnoreCase(createURI(value).getScheme()); }
private String fetchFilesNotInLocalFilesystem(String cmd) { SessionState ss = SessionState.get(); String progName = getScriptProgName(cmd); if (!ResourceDownloader.isFileUri(progName)) { String filePath = ss.add_resource(ResourceType.FILE, progName, true); Path p = new Path(filePath); String fileName = p.getName(); String scriptArgs = getScriptArgs(cmd); String finalCmd = fileName + scriptArgs; return finalCmd; } return cmd; }
public List<URI> downloadExternal(URI source, String subDir, boolean convertToUnix) throws URISyntaxException, IOException { return resolveAndDownloadInternal(source, subDir, convertToUnix, false); }