/** * Gets the set of resources information that are from the Spark Streaming Core. It excludes any * Spark streaming extensions, such as Kafka or Flume. They need to be excluded since they are not * part of Spark distribution and it should be loaded from the user program ClassLoader. This filtering * is needed for unit-testing because in unit-test, those extension classes are loadable from the system * classloader, causing same classes being loaded through different classloader. */ private Set<ClassPath.ResourceInfo> getSparkStreamingResources() { if (sparkStreamingResources != null) { return sparkStreamingResources; } synchronized (this) { if (sparkStreamingResources != null) { return sparkStreamingResources; } try { sparkStreamingResources = ClassPathResources.getClassPathResources(getClass().getClassLoader(), StreamingContext.class); } catch (IOException e) { LOG.warn("Failed to find resources for Spark StreamingContext.", e); sparkStreamingResources = Collections.emptySet(); } return sparkStreamingResources; } } };
private static Set<String> traceSecurityDependencies(ClassLoader baseClassLoader) { try { // Trace dependencies for Authorizer class. This will make classes from cdap-security-spi as well as cdap-proto // and other dependencies of cdap-security-spi available to the authorizer extension. return ClassPathResources.getResourcesWithDependencies(baseClassLoader, Authorizer.class); } catch (IOException e) { LOG.error("Failed to determine resources for authorizer class loader while tracing dependencies of " + "Authorizer.", e); return ImmutableSet.of(); } }
/** * Returns a set of {@link ResourceInfo} required for the specified {@link Class} using the specified * {@link ClassLoader}. Does not trace dependencies of the specified class. * * @param classLoader the {@link ClassLoader} to use to return the set of {@link ResourceInfo} for the specified * {@link Class} * @param cls the {@link Class} for which to return the set of {@link ResourceInfo} * @return the set of {@link ResourceInfo} required for the specified {@link Class} using the specified * {@link ClassLoader} */ public static Set<ClassPath.ResourceInfo> getClassPathResources(ClassLoader classLoader, Class<?> cls) throws IOException { return getClassPath(classLoader, cls).getResources(); }
/** * Returns a Set of resources name that are visible through the cdap-api module as well as Hadoop classes. * This includes all classes+resources in cdap-api plus all classes+resources that cdap-api * depends on (for example, sl4j, gson, etc). */ private static Set<String> createBaseResources() throws IOException { // Everything should be traceable in the same ClassLoader of this class, which is the CDAP system ClassLoader ClassLoader classLoader = ProgramResources.class.getClassLoader(); // Gather resources information for cdap-api classes // Add everything in cdap-api as visible resources // Trace dependencies for cdap-api classes Set<String> result = ClassPathResources.getResourcesWithDependencies(classLoader, Application.class); // Gather resources for javax.ws.rs classes. They are not traceable from the api classes. Iterables.addAll(result, Iterables.transform(ClassPathResources.getClassPathResources(classLoader, Path.class), ClassPathResources.RESOURCE_INFO_TO_RESOURCE_NAME)); // Gather Hadoop classes and resources getResources(ClassPath.from(classLoader, JAR_ONLY_URI), HADOOP_PACKAGES, EXCLUDE_PACKAGES, ClassPathResources.RESOURCE_INFO_TO_RESOURCE_NAME, result); return Collections.unmodifiableSet(result); }
/** * Returns the base set of resources needed to load the specified {@link Class} using the * specified {@link ClassLoader}. Also traces and includes the dependencies for the specified class. * * @param classLoader the {@link ClassLoader} to use to generate the set of resources * @param classz the {@link Class} to generate the set of resources for * @return the set of resources needed to load the specified {@link Class} using the specified {@link ClassLoader} * @throws IOException */ public static Set<String> getResourcesWithDependencies(ClassLoader classLoader, Class<?> classz) throws IOException { ClassPath classPath = getClassPath(classLoader, classz); // Add everything in the classpath as visible resources Set<String> result = Sets.newHashSet(Iterables.transform(classPath.getResources(), RESOURCE_INFO_TO_RESOURCE_NAME)); // Trace dependencies for all classes in the classpath findClassDependencies( classLoader, Iterables.transform(classPath.getAllClasses(), CLASS_INFO_TO_CLASS_NAME), result); return result; }
/** * Returns a {@link ClassPath} instance that represents the classpath that the given class is loaded from the given * ClassLoader. */ private static ClassPath getClassPath(ClassLoader classLoader, Class<?> cls) throws IOException { String resourceName = cls.getName().replace('.', '/') + ".class"; URL url = classLoader.getResource(resourceName); if (url == null) { throw new IOException("Resource not found for " + resourceName); } try { URI classPathURI = getClassPathURL(resourceName, url).toURI(); return ClassPath.from(classPathURI, classLoader); } catch (URISyntaxException e) { throw new IOException(e); } }
Iterable<String> classes, final T result) throws IOException { final Set<String> bootstrapClassPaths = getBootstrapClassPaths(); final Set<URL> classPathSeen = Sets.newHashSet();
/** * Returns a Set of resources name that are visible through the cdap-api module as well as Hadoop classes. * This includes all classes+resources in cdap-api plus all classes+resources that cdap-api * depends on (for example, sl4j, gson, etc). */ private static Set<String> createBaseResources() throws IOException { // Everything should be traceable in the same ClassLoader of this class, which is the CDAP system ClassLoader ClassLoader classLoader = ProgramResources.class.getClassLoader(); // Gather resources information for cdap-api classes // Add everything in cdap-api as visible resources // Trace dependencies for cdap-api classes Set<String> result = ClassPathResources.getResourcesWithDependencies(classLoader, Application.class); // Gather resources for javax.ws.rs classes. They are not traceable from the api classes. Iterables.addAll(result, Iterables.transform(ClassPathResources.getClassPathResources(classLoader, Path.class), ClassPathResources.RESOURCE_INFO_TO_RESOURCE_NAME)); // Gather Hadoop classes and resources getResources(ClassPath.from(classLoader, JAR_ONLY_URI), HADOOP_PACKAGES, EXCLUDE_PACKAGES, ClassPathResources.RESOURCE_INFO_TO_RESOURCE_NAME, result); return Collections.unmodifiableSet(result); }
/** * Returns the base set of resources needed to load the specified {@link Class} using the * specified {@link ClassLoader}. Also traces and includes the dependencies for the specified class. * * @param classLoader the {@link ClassLoader} to use to generate the set of resources * @param classz the {@link Class} to generate the set of resources for * @return the set of resources needed to load the specified {@link Class} using the specified {@link ClassLoader} * @throws IOException */ public static Set<String> getResourcesWithDependencies(ClassLoader classLoader, Class<?> classz) throws IOException { ClassPath classPath = getClassPath(classLoader, classz); // Add everything in the classpath as visible resources Set<String> result = Sets.newHashSet(Iterables.transform(classPath.getResources(), RESOURCE_INFO_TO_RESOURCE_NAME)); // Trace dependencies for all classes in the classpath findClassDependencies( classLoader, Iterables.transform(classPath.getAllClasses(), CLASS_INFO_TO_CLASS_NAME), result); return result; }
/** * Returns a {@link ClassPath} instance that represents the classpath that the given class is loaded from the given * ClassLoader. */ private static ClassPath getClassPath(ClassLoader classLoader, Class<?> cls) throws IOException { String resourceName = cls.getName().replace('.', '/') + ".class"; URL url = classLoader.getResource(resourceName); if (url == null) { throw new IOException("Resource not found for " + resourceName); } try { URI classPathURI = getClassPathURL(resourceName, url).toURI(); return ClassPath.from(classPathURI, classLoader); } catch (URISyntaxException e) { throw new IOException(e); } }
Iterable<String> classes, final T result) throws IOException { final Set<String> bootstrapClassPaths = getBootstrapClassPaths(); final Set<URL> classPathSeen = Sets.newHashSet();
/** * Gets the set of resources information that are from the Spark Streaming Core. It excludes any * Spark streaming extensions, such as Kafka or Flume. They need to be excluded since they are not * part of Spark distribution and it should be loaded from the user program ClassLoader. This filtering * is needed for unit-testing because in unit-test, those extension classes are loadable from the system * classloader, causing same classes being loaded through different classloader. */ private Set<ClassPath.ResourceInfo> getSparkStreamingResources() { if (sparkStreamingResources != null) { return sparkStreamingResources; } synchronized (this) { if (sparkStreamingResources != null) { return sparkStreamingResources; } try { sparkStreamingResources = ClassPathResources.getClassPathResources(getClass().getClassLoader(), StreamingContext.class); } catch (IOException e) { LOG.warn("Failed to find resources for Spark StreamingContext.", e); sparkStreamingResources = Collections.emptySet(); } return sparkStreamingResources; } } };
/** * Returns a set of {@link ResourceInfo} required for the specified {@link Class} using the specified * {@link ClassLoader}. Does not trace dependencies of the specified class. * * @param classLoader the {@link ClassLoader} to use to return the set of {@link ResourceInfo} for the specified * {@link Class} * @param cls the {@link Class} for which to return the set of {@link ResourceInfo} * @return the set of {@link ResourceInfo} required for the specified {@link Class} using the specified * {@link ClassLoader} */ public static Set<ClassPath.ResourceInfo> getClassPathResources(ClassLoader classLoader, Class<?> cls) throws IOException { return getClassPath(classLoader, cls).getResources(); }
/** * Gets the set of resources information that are from the Spark Streaming Core. It excludes any * Spark streaming extensions, such as Kafka or Flume. They need to be excluded since they are not * part of Spark distribution and it should be loaded from the user program ClassLoader. This filtering * is needed for unit-testing because in unit-test, those extension classes are loadable from the system * classloader, causing same classes being loaded through different classloader. */ private Set<ClassPath.ResourceInfo> getSparkStreamingResources() { if (sparkStreamingResources != null) { return sparkStreamingResources; } synchronized (this) { if (sparkStreamingResources != null) { return sparkStreamingResources; } try { sparkStreamingResources = ClassPathResources.getClassPathResources(getClass().getClassLoader(), StreamingContext.class); } catch (IOException e) { LOG.warn("Failed to find resources for Spark StreamingContext.", e); sparkStreamingResources = Collections.emptySet(); } return sparkStreamingResources; } } };