/** * Creates a program {@link ClassLoader} based on the MR job config. */ private static ClassLoader createProgramClassLoader(MapReduceContextConfig contextConfig) { // In distributed mode, the program is created by expanding the program jar. // The program jar is localized to container with the program jar name. // It's ok to expand to a temp dir in local directory, as the YARN container will be gone. Location programLocation = Locations.toLocation(new File(contextConfig.getProgramJarName())); try { File unpackDir = DirUtils.createTempDir(new File(System.getProperty("user.dir"))); LOG.info("Create ProgramClassLoader from {}, expand to {}", programLocation, unpackDir); BundleJarUtil.unJar(programLocation, unpackDir); return new ProgramClassLoader(contextConfig.getCConf(), unpackDir, FilterClassLoader.create(contextConfig.getHConf().getClassLoader())); } catch (IOException e) { LOG.error("Failed to create ProgramClassLoader", e); throw Throwables.propagate(e); } }
/** * Creates a program {@link ClassLoader} based on the MR job config. */ private static ClassLoader createProgramClassLoader(MapReduceContextConfig contextConfig) { // In distributed mode, the program is created by expanding the program jar. // The program jar is localized to container with the program jar name. // It's ok to expand to a temp dir in local directory, as the YARN container will be gone. Location programLocation = Locations.toLocation(new File(contextConfig.getProgramJarName())); try { File unpackDir = DirUtils.createTempDir(new File(System.getProperty("user.dir"))); LOG.info("Create ProgramClassLoader from {}, expand to {}", programLocation, unpackDir); BundleJarUtil.unJar(programLocation, unpackDir); return new ProgramClassLoader(contextConfig.getCConf(), unpackDir, FilterClassLoader.create(contextConfig.getHConf().getClassLoader())); } catch (IOException e) { LOG.error("Failed to create ProgramClassLoader", e); throw Throwables.propagate(e); } }
/** * Creates a {@link Program} instance based on the information from the {@link MapReduceContextConfig}, using * the given program ClassLoader. */ private Program createProgram(MapReduceContextConfig contextConfig, ClassLoader programClassLoader) { Location programLocation; LocationFactory locationFactory = new LocalLocationFactory(); // Use the program jar location regardless if local or distributed, since it is valid for both programLocation = locationFactory.create(new File(contextConfig.getProgramJarName()).getAbsoluteFile().toURI()); return new DefaultProgram(new ProgramDescriptor(contextConfig.getProgramId(), contextConfig.getApplicationSpecification()), programLocation, programClassLoader); }
/** * Creates a {@link Program} instance based on the information from the {@link MapReduceContextConfig}, using * the given program ClassLoader. */ private Program createProgram(MapReduceContextConfig contextConfig, ClassLoader programClassLoader) { Location programLocation; LocationFactory locationFactory = new LocalLocationFactory(); if (isLocal(contextConfig.getHConf())) { // Just create a local location factory. It's for temp usage only as the program location is always absolute. programLocation = locationFactory.create(contextConfig.getProgramJarURI()); } else { // In distributed mode, the program jar is localized to the container programLocation = locationFactory.create(new File(contextConfig.getProgramJarName()).getAbsoluteFile().toURI()); } return new DefaultProgram(new ProgramDescriptor(contextConfig.getProgramId(), contextConfig.getApplicationSpecification()), programLocation, programClassLoader); }