/** * Copy event handler from current class configuration into runtime injector. * This is a helper method called from initializeInjector() only. * @param runtimeInjector Runtime injector to copy event handler to. * @param param Class that identifies the event handler parameter. * @param <T> Type of the event handler. * @throws InjectionException If configuration error occurs. */ private <T extends EventHandler<?>> void copyEventHandler( final Injector runtimeInjector, final Class<? extends Name<T>> param) throws InjectionException { runtimeInjector.bindVolatileParameter(param, this.originalInjector.getNamedInstance(param)); }
/** * Run user-provided main method. * * @param jobConf the job configuration * @throws Exception on any exceptions on the way */ private static void runUserProgramMain(final Configuration jobConf) throws Exception { final Injector injector = TANG.newInjector(jobConf); final String className = injector.getNamedInstance(JobConf.UserMainClass.class); final String userArgsString = injector.getNamedInstance(JobConf.UserMainArguments.class); final String[] args = userArgsString.isEmpty() ? EMPTY_USER_ARGS : userArgsString.split(" "); final Class userCode = Class.forName(className); final Method method = userCode.getMethod("main", String[].class); if (!Modifier.isStatic(method.getModifiers())) { throw new RuntimeException("User Main Method not static"); } if (!Modifier.isPublic(userCode.getModifiers())) { throw new RuntimeException("User Main Class not public"); } LOG.info("User program started"); method.invoke(null, (Object) args); LOG.info("User program finished"); }
/** * Extracts a task id from the given configuration. * * @param config * @return the task id in the given configuration. * @throws RuntimeException if the configuration can't be parsed. */ public static String getTaskId(final Configuration config) { try { return Tang.Factory.getTang().newInjector(config).getNamedInstance(TaskConfigurationOptions.Identifier.class); } catch (final InjectionException ex) { throw new RuntimeException("Unable to determine task identifier. Giving up.", ex); } }
/** * Extracts a context id from the given configuration. * * @param c * @return the context id in the given configuration. * @throws RuntimeException if the configuration can't be parsed. */ public static String getIdentifier(final Configuration c) { try { return Tang.Factory.getTang().newInjector(c).getNamedInstance( ContextIdentifier.class); } catch (final InjectionException e) { throw new RuntimeException("Unable to determine context identifier. Giving up.", e); } }
/** * Extracts the queue name from the driverConfiguration or return default if none is set. * * @param driverConfiguration The drievr configuration * @return the queue name from the driverConfiguration or return default if none is set. */ private String getQueue(final Configuration driverConfiguration) { try { return Tang.Factory.getTang().newInjector(driverConfiguration).getNamedInstance(JobQueue.class); } catch (final InjectionException e) { return this.defaultQueueName; } }
private String taskId(final Configuration partialTaskConf) { try { final Injector injector = Tang.Factory.getTang().newInjector(partialTaskConf); return injector.getNamedInstance(TaskConfigurationOptions.Identifier.class); } catch (final InjectionException e) { throw new RuntimeException(getQualifiedName() + "Injection exception while extracting taskId from partialTaskConf", e); } }
private static DAG<IRVertex, IREdge> compileDAG(final String[] args) throws Exception { final String userMainClassName; final String[] userMainMethodArgs; try { final Configuration configuration = JobLauncher.getJobConf(args); final Injector injector = Tang.Factory.getTang().newInjector(configuration); userMainClassName = injector.getNamedInstance(JobConf.UserMainClass.class); userMainMethodArgs = injector.getNamedInstance(JobConf.UserMainArguments.class).split(" "); } catch (final Exception e) { throw new RuntimeException("An exception occurred while processing configuration for invoking user main. " + "Note: Using compileDAG for multiple times will fail, as compileDAG method enables static method mocking " + "on JobLauncher and because of this Tang may misbehave afterwards.", e); } final Class userMainClass = Class.forName(userMainClassName); final Method userMainMethod = userMainClass.getMethod("main", String[].class); final ArgumentCaptor<DAG> captor = ArgumentCaptor.forClass(DAG.class); final ArgumentCaptor<String> stringArg = ArgumentCaptor.forClass(String.class); PowerMockito.mockStatic(JobLauncher.class); PowerMockito.doNothing().when(JobLauncher.class, "launchDAG", captor.capture(), stringArg.capture()); userMainMethod.invoke(null, (Object) userMainMethodArgs); return captor.getValue(); }
/** * Extract the queue name from the jobSubmissionEvent or return default if none is set. */ private String getQueue(final JobSubmissionEvent jobSubmissionEvent) { try { return Tang.Factory.getTang().newInjector( jobSubmissionEvent.getConfiguration()).getNamedInstance(JobQueue.class); } catch (final InjectionException e) { return this.defaultQueueName; } } }
/** * Read json file and return its contents as configuration parameter. * * @param jobConf job configuration to get json path. * @param pathParameter named parameter represents path to the json file, or an empty string * @param contentsParameter named parameter represents contents of the file * @param defaultContent the default configuration * @return configuration with contents of the file, or an empty string as value for {@code contentsParameter} * @throws InjectionException exception while injection. */ private static Configuration getJSONConf(final Configuration jobConf, final Class<? extends Name<String>> pathParameter, final Class<? extends Name<String>> contentsParameter, final String defaultContent) throws InjectionException { final Injector injector = TANG.newInjector(jobConf); try { final String path = injector.getNamedInstance(pathParameter); final String contents = path.isEmpty() ? defaultContent : new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8); return TANG.newConfigurationBuilder() .bindNamedParameter(contentsParameter, contents) .build(); } catch (final IOException e) { throw new RuntimeException(e); } }
/** * Get deploy mode configuration. * * @param jobConf job configuration to get deploy mode. * @return deploy mode configuration. * @throws InjectionException exception while injection. */ private static Configuration getDeployModeConf(final Configuration jobConf) throws InjectionException { final Injector injector = TANG.newInjector(jobConf); final String deployMode = injector.getNamedInstance(JobConf.DeployMode.class); switch (deployMode) { case "local": return LocalRuntimeConfiguration.CONF .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, LOCAL_NUMBER_OF_EVALUATORS) .build(); case "yarn": return YarnClientConfiguration.CONF .set(YarnClientConfiguration.JVM_HEAP_SLACK, injector.getNamedInstance(JobConf.JVMHeapSlack.class)) .build(); default: throw new UnsupportedOperationException(deployMode); } }
final Injector injector = Tang.Factory.getTang().newInjector(driverConfiguration); final boolean preserveEvaluators = injector.getNamedInstance(ResourceManagerPreserveEvaluators.class); final int maxAppSubmissions = injector.getNamedInstance(MaxApplicationSubmissions.class); .setIdentifier(returnOrGenerateDriverId(injector.getNamedInstance(DriverIdentifier.class))) .setDriverMemory(injector.getNamedInstance(DriverMemory.class)) .setUserName(System.getProperty("user.name")) .setPreserveEvaluators(preserveEvaluators) .setConfiguration(driverConfiguration); for (final String globalFileName : injector.getNamedInstance(JobGlobalFiles.class)) { LOG.log(Level.FINEST, "Adding global file: {0}", globalFileName); jbuilder.addGlobalFile(getFileResourceProto(globalFileName, FileType.PLAIN)); for (final String globalLibraryName : injector.getNamedInstance(JobGlobalLibraries.class)) { LOG.log(Level.FINEST, "Adding global library: {0}", globalLibraryName); jbuilder.addGlobalFile(getFileResourceProto(globalLibraryName, FileType.LIB)); for (final String localFileName : injector.getNamedInstance(DriverLocalFiles.class)) { LOG.log(Level.FINEST, "Adding local file: {0}", localFileName); jbuilder.addLocalFile(getFileResourceProto(localFileName, FileType.PLAIN)); for (final String localLibraryName : injector.getNamedInstance(DriverLocalLibraries.class)) { LOG.log(Level.FINEST, "Adding local library: {0}", localLibraryName); jbuilder.addLocalFile(getFileResourceProto(localLibraryName, FileType.LIB));
private static Optional<String> getUserBoundJobSubmissionDirectory(final Configuration configuration) { try { return Optional.ofNullable(Tang.Factory.getTang().newInjector(configuration) .getNamedInstance(DriverJobSubmissionDirectory.class)); } catch (final InjectionException ex) { return Optional.empty(); } }
/** * Fetch scheduler configuration. * @param jobConf job configuration. * @return the scheduler configuration. * @throws ClassNotFoundException exception while finding the class. * @throws InjectionException exception while injection (REEF Tang). */ private static Configuration getSchedulerConf(final Configuration jobConf) throws ClassNotFoundException, InjectionException { final Injector injector = TANG.newInjector(jobConf); final String classImplName = injector.getNamedInstance(JobConf.SchedulerImplClassName.class); final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder(); final Class schedulerImpl = ((Class<Scheduler>) Class.forName(classImplName)); jcb.bindImplementation(Scheduler.class, schedulerImpl); return jcb.build(); }
@Override public SparkSession getOrCreate() { if (!options.containsKey("spark.master")) { // default spark_master option. return this.master("local[*]").getOrCreate(); } if (!options.containsKey("spark.driver.allowMultipleContexts")) { return this.config("spark.driver.allowMultipleContexts", "true").getOrCreate(); } UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser("ubuntu")); // Set up spark context with given options. final SparkConf sparkConf = new SparkConf(); if (!options.containsKey("spark.app.name")) { try { // get and override configurations from JobLauncher. final Configuration configurations = JobLauncher.getBuiltJobConf(); final Injector injector = Tang.Factory.getTang().newInjector(configurations); options.put("spark.app.name", injector.getNamedInstance(JobConf.JobId.class)); } catch (final InjectionException e) { throw new RuntimeException(e); } } options.forEach(sparkConf::set); final SparkContext sparkContext = new org.apache.nemo.compiler.frontend.spark.core.SparkContext(sparkConf); super.sparkContext(sparkContext); return SparkSession.from(super.getOrCreate(), this.options); } }
final String operName = forkedInjector.getNamedInstance(OperatorName.class); this.operators.put(Utils.getClass(operName), operator); LOG.finest(operName + " has CommGroupHandler-" + commGroupNetworkHandler.toString());
/** * Get driver configuration. * * @param jobConf job Configuration to get job id and driver memory. * @return driver configuration. * @throws InjectionException exception while injection. */ private static Configuration getDriverConf(final Configuration jobConf) throws InjectionException { final Injector injector = TANG.newInjector(jobConf); final String jobId = injector.getNamedInstance(JobConf.JobId.class); final int driverMemory = injector.getNamedInstance(JobConf.DriverMemMb.class); return DriverConfiguration.CONF .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(NemoDriver.class)) .set(DriverConfiguration.ON_DRIVER_STARTED, NemoDriver.StartHandler.class) .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, NemoDriver.AllocatedEvaluatorHandler.class) .set(DriverConfiguration.ON_CONTEXT_ACTIVE, NemoDriver.ActiveContextHandler.class) .set(DriverConfiguration.ON_EVALUATOR_FAILED, NemoDriver.FailedEvaluatorHandler.class) .set(DriverConfiguration.ON_CONTEXT_FAILED, NemoDriver.FailedContextHandler.class) .set(DriverConfiguration.ON_DRIVER_STOP, NemoDriver.DriverStopHandler.class) .set(DriverConfiguration.DRIVER_IDENTIFIER, jobId) .set(DriverConfiguration.DRIVER_MEMORY, driverMemory) .build(); }
/** * Create a new ContextRuntime. * * @param serviceInjector the serviceInjector to be used. * @param contextConfiguration the Configuration for this context. * @throws ContextClientCodeException if the context cannot be instantiated. */ ContextRuntime(final Injector serviceInjector, final Configuration contextConfiguration, final Optional<ContextRuntime> parentContext) throws ContextClientCodeException { this.serviceInjector = serviceInjector; this.parentContext = parentContext; // Trigger the instantiation of the services try { final Set<Object> services = serviceInjector.getNamedInstance(Services.class); this.contextInjector = serviceInjector.forkInjector(contextConfiguration); this.contextLifeCycle = this.contextInjector.getInstance(ContextLifeCycle.class); } catch (BindException | InjectionException e) { final Optional<String> parentID = this.getParentContext().isPresent() ? Optional.of(this.getParentContext().get().getIdentifier()) : Optional.<String>empty(); throw new ContextClientCodeException( ContextClientCodeException.getIdentifier(contextConfiguration), parentID, "Unable to spawn context", e); } // Trigger the context start events on contextInjector. this.contextLifeCycle.start(); }