private void addJarLRByClass(Class<?> clazz, final Map<String, LocalResource> lrMap) throws IOException, LoginException { final File jar = new File(Utilities.jarFinderGetJar(clazz)); final String localJarPath = jar.toURI().toURL().toExternalForm(); final LocalResource jarLr = createJarLocalResource(localJarPath); lrMap.put(DagUtils.getBaseName(jarLr), jarLr); }
private void addJarLRByClass(Class<?> clazz, final Map<String, LocalResource> lrMap) throws IOException, LoginException { final File jar = new File(Utilities.jarFinderGetJar(clazz)); final LocalResource jarLr = createJarLocalResource(jar.toURI().toURL().toExternalForm()); lrMap.put(utils.getBaseName(jarLr), jarLr); }
@SuppressWarnings("SameParameterValue") static void addDependencyJars(Configuration conf, Class<?>... classes) throws IOException { FileSystem localFs = FileSystem.getLocal(conf); Set<String> jars = new HashSet<>(conf.getStringCollection("tmpjars")); for (Class<?> clazz : classes) { if (clazz == null) { continue; } final String path = Utilities.jarFinderGetJar(clazz); if (path == null) { throw new RuntimeException("Could not find jar for class " + clazz + " in order to ship it to the cluster."); } if (!localFs.exists(new Path(path))) { throw new RuntimeException("Could not validate jar file " + path + " for class " + clazz); } jars.add(path); } if (jars.isEmpty()) { return; } //noinspection ToArrayCallWithZeroLengthArrayArgument conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()]))); }
private void localizeJarForClass(FileSystem lfs, Path libDir, String className, boolean doThrow) throws IOException { String jarPath = null; boolean hasException = false; try { Class<?> auxClass = Class.forName(className); jarPath = Utilities.jarFinderGetJar(auxClass); } catch (Throwable t) { if (doThrow) { throw (t instanceof IOException) ? (IOException)t : new IOException(t); } hasException = true; String err = "Cannot find a jar for [" + className + "] due to an exception (" + t.getMessage() + "); not packaging the jar"; LOG.error(err); System.err.println(err); } if (jarPath != null) { lfs.copyFromLocalFile(new Path(jarPath), libDir); } else if (!hasException) { String err = "Cannot find a jar for [" + className + "]; not packaging the jar"; if (doThrow) { throw new IOException(err); } LOG.error(err); System.err.println(err); } }
@SuppressWarnings("SameParameterValue") static void copyDependencyJars(Configuration conf, Class<?>... classes) throws IOException { Set<String> jars = new HashSet<>(); FileSystem localFs = FileSystem.getLocal(conf); jars.addAll(conf.getStringCollection("tmpjars")); jars.addAll(Arrays.stream(classes) .filter(Objects::nonNull) .map(clazz -> { String path = Utilities.jarFinderGetJar(clazz); if (path == null) { throw new RuntimeException("Could not find jar for class " + clazz + " in order to ship it to the cluster."); } try { if (!localFs.exists(new Path(path))) { throw new RuntimeException("Could not validate jar file " + path + " for class " + clazz); } } catch (IOException e) { throw new RuntimeException(e); } return path; }).collect(Collectors.toList())); if (jars.isEmpty()) { return; } conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0]))); }
Path jarPath = new Path(Utilities.jarFinderGetJar(c)); lfs.copyFromLocalFile(jarPath, libDir); if (LOG.isDebugEnabled()) {
private void localizeJarForClass(FileSystem lfs, Path libDir, String className, boolean doThrow) throws IOException { String jarPath = null; boolean hasException = false; try { Class<?> auxClass = Class.forName(className); jarPath = Utilities.jarFinderGetJar(auxClass); } catch (Throwable t) { if (doThrow) { throw (t instanceof IOException) ? (IOException)t : new IOException(t); } hasException = true; String err = "Cannot find a jar for [" + className + "] due to an exception (" + t.getMessage() + "); not packaging the jar"; LOG.error(err); System.err.println(err); } if (jarPath != null) { lfs.copyFromLocalFile(new Path(jarPath), libDir); } else if (!hasException) { String err = "Cannot find a jar for [" + className + "]; not packaging the jar"; if (doThrow) { throw new IOException(err); } LOG.error(err); System.err.println(err); } }
@SuppressWarnings("SameParameterValue") static void copyDependencyJars(Configuration conf, Class<?>... classes) throws IOException { Set<String> jars = new HashSet<>(); FileSystem localFs = FileSystem.getLocal(conf); jars.addAll(conf.getStringCollection("tmpjars")); jars.addAll(Arrays.stream(classes) .filter(Objects::nonNull) .map(clazz -> { String path = Utilities.jarFinderGetJar(clazz); if (path == null) { throw new RuntimeException("Could not find jar for class " + clazz + " in order to ship it to the cluster."); } try { if (!localFs.exists(new Path(path))) { throw new RuntimeException("Could not validate jar file " + path + " for class " + clazz); } } catch (IOException e) { throw new RuntimeException(e); } return path; }).collect(Collectors.toList())); if (jars.isEmpty()) { return; } conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0]))); }
Path jarPath = new Path(Utilities.jarFinderGetJar(c)); lfs.copyFromLocalFile(jarPath, libDir); if (LOG.isDebugEnabled()) {