/** * Get driver ncs configuration. * * @return driver ncs configuration. * @throws InjectionException exception while injection. */ private static Configuration getDriverNcsConf() throws InjectionException { return Configurations.merge(NameServerConfiguration.CONF.build(), LocalNameResolverConfiguration.CONF.build(), TANG.newConfigurationBuilder() .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class) .build()); }
public static void main(final String[] args) throws BindException, InjectionException, InterruptedException { final Configuration c = TimerMock.CONF .set(TimerMockConf.MOCK_SLEEP_TIME, 1) .build(); final Timer t = Tang.Factory.getTang().newInjector(c).getInstance(Timer.class); System.out.println("Tick..."); t.sleep(); System.out.println("...tock."); }
@Override public void submitTask(final Configuration taskConfiguration) { final Configuration contextConfiguration = ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, "RootContext_" + this.getId()) .build(); this.submitContextAndTask(contextConfiguration, taskConfiguration); }
@Override public Configuration getContextConfiguration() { LOG.entering("GroupCommDriverImpl", "getContextConf"); final Configuration retVal = ContextConfiguration.CONF.set(ContextConfiguration.IDENTIFIER, "GroupCommunicationContext-" + contextIds.getAndIncrement()).build(); LOG.exiting("GroupCommDriverImpl", "getContextConf", confSerializer.toString(retVal)); return retVal; }
/** * Get deploy mode configuration. * * @param jobConf job configuration to get deploy mode. * @return deploy mode configuration. * @throws InjectionException exception while injection. */ private static Configuration getDeployModeConf(final Configuration jobConf) throws InjectionException { final Injector injector = TANG.newInjector(jobConf); final String deployMode = injector.getNamedInstance(JobConf.DeployMode.class); switch (deployMode) { case "local": return LocalRuntimeConfiguration.CONF .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, LOCAL_NUMBER_OF_EVALUATORS) .build(); case "yarn": return YarnClientConfiguration.CONF .set(YarnClientConfiguration.JVM_HEAP_SLACK, injector.getNamedInstance(JobConf.JVMHeapSlack.class)) .build(); default: throw new UnsupportedOperationException(deployMode); } }
private Configuration getYarnConfiguration(final URI jobFolder, final String clientRemoteId, final String jobId) { return YarnDriverConfiguration.CONF .set(YarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, jobFolder.toString()) .set(YarnDriverConfiguration.JOB_IDENTIFIER, jobId) .set(YarnDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId) .set(YarnDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack) .set(YarnDriverConfiguration.RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME) .build(); }
@Override public Configuration getDriverConfiguration(final URI jobFolder, final String clientRemoteId, final String jobId, final Configuration applicationConfiguration) { return Configurations.merge( org.apache.reef.runtime.yarn.driver.YarnDriverConfiguration.CONF .set(JOB_SUBMISSION_DIRECTORY, jobFolder.toString()) .set(JOB_IDENTIFIER, jobId) .set(CLIENT_REMOTE_IDENTIFIER, clientRemoteId) .set(JVM_HEAP_SLACK, this.jvmSlack) .set(RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME) .build(), applicationConfiguration); } }
@Override public Configuration getDriverConfiguration(final URI jobFolder, final String clientRemoteId, final String jobId, final Configuration applicationConfiguration) { final Configuration hdinsightDriverConfiguration = HDInsightDriverConfiguration.CONF .set(HDInsightDriverConfiguration.JOB_IDENTIFIER, jobId) .set(HDInsightDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId) .set(HDInsightDriverConfiguration.JOB_SUBMISSION_DIRECTORY, jobFolder.toString()) .set(HDInsightDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack) .set(HDInsightDriverConfiguration.RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME) .build(); return Configurations.merge( applicationConfiguration, hdinsightDriverConfiguration); } }
private Configuration getDriverConfiguration(final URI jobFolder, final String clientRemoteId, final String jobId) { ConfigurationModule configModule = LocalDriverConfiguration.CONF .set(LocalDriverConfiguration.MAX_NUMBER_OF_EVALUATORS, this.maxEvaluators) .set(LocalDriverConfiguration.ROOT_FOLDER, jobFolder.getPath()) .set(LocalDriverConfiguration.JVM_HEAP_SLACK, this.jvmHeapSlack) .set(LocalDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId) .set(LocalDriverConfiguration.JOB_IDENTIFIER, jobId) .set(LocalDriverConfiguration.RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME); for (final String rackName : rackNames) { configModule = configModule.set(LocalDriverConfiguration.RACK_NAMES, rackName); } return configModule.build(); }
ExtensibleYarnClientConfiguration.CONF .set(ExtensibleYarnClientConfiguration.DRIVER_CONFIGURATION_PROVIDER, MultiRuntimeDriverConfigurationProvider.class).build());
private Configuration getExecutorConfiguration(final String executorId) { final Configuration executorConfiguration = JobConf.EXECUTOR_CONF .set(JobConf.EXECUTOR_ID, executorId) .set(JobConf.GLUSTER_DISK_DIRECTORY, glusterDirectory) .set(JobConf.LOCAL_DISK_DIRECTORY, localDirectory) .set(JobConf.JOB_ID, jobId) .build(); final Configuration contextConfiguration = ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, executorId) // We set: contextId = executorId .set(ContextConfiguration.ON_CONTEXT_STARTED, NemoContext.ContextStartHandler.class) .set(ContextConfiguration.ON_CONTEXT_STOP, NemoContext.ContextStopHandler.class) .build(); final Configuration ncsConfiguration = getExecutorNcsConfiguration(); final Configuration messageConfiguration = getExecutorMessageConfiguration(executorId); return Configurations.merge(executorConfiguration, contextConfiguration, ncsConfiguration, messageConfiguration); }
private Configuration getExecutorConfiguration(final String executorId) { final Configuration executorConfiguration = JobConf.EXECUTOR_CONF .set(JobConf.EXECUTOR_ID, executorId) .set(JobConf.GLUSTER_DISK_DIRECTORY, glusterDirectory) .set(JobConf.LOCAL_DISK_DIRECTORY, localDirectory) .set(JobConf.JOB_ID, jobId) .build(); final Configuration contextConfiguration = ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, executorId) // We set: contextId = executorId .set(ContextConfiguration.ON_CONTEXT_STARTED, NemoContext.ContextStartHandler.class) .set(ContextConfiguration.ON_CONTEXT_STOP, NemoContext.ContextStopHandler.class) .build(); final Configuration ncsConfiguration = getExecutorNcsConfiguration(); final Configuration messageConfiguration = getExecutorMessageConfiguration(executorId); return Configurations.merge(executorConfiguration, contextConfiguration, ncsConfiguration, messageConfiguration); }
@Override public Configuration getContextConfiguration(final AllocatedEvaluator allocatedEvaluator) { final NumberedSplit<InputSplit> numberedSplit = this.evaluatorToPartitionStrategy.getInputSplit( allocatedEvaluator.getEvaluatorDescriptor().getNodeDescriptor(), allocatedEvaluator.getId()); return ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, DATA_LOAD_CONTEXT_PREFIX + numberedSplit.getIndex()) .build(); }
/** * Submit Task with configuration strings. * This method should be called from bridge and the configuration strings are * serialized at .Net side. * @param evaluatorConfiguration * @param taskConfiguration */ public void submitTask(final String evaluatorConfiguration, final String taskConfiguration) { final Configuration contextConfiguration = ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, "RootContext_" + this.getId()) .build(); final String contextConfigurationString = this.configurationSerializer.toString(contextConfiguration); this.launchWithConfigurationString( evaluatorConfiguration, contextConfigurationString, Optional.<String>empty(), Optional.of(taskConfiguration)); }
private Configuration getLocalConfiguration(final URI jobFolder, final String clientRemoteId, final String jobId) { ConfigurationModule localModule = LocalDriverConfiguration.CONF .set(LocalDriverConfiguration.MAX_NUMBER_OF_EVALUATORS, this.maxEvaluators) // ROOT FOLDER will point to the current runtime directory .set(LocalDriverConfiguration.ROOT_FOLDER, ".") .set(LocalDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack) .set(LocalDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId) .set(LocalDriverConfiguration.JOB_IDENTIFIER, jobId) .set(LocalDriverConfiguration.RUNTIME_NAMES, org.apache.reef.runtime.local.driver.RuntimeIdentifier.RUNTIME_NAME); for (final String rackName : rackNames) { localModule = localModule.set(LocalDriverConfiguration.RACK_NAMES, rackName); } return localModule.build(); }
@Override public Configuration getDriverConfiguration(final URI jobFolder, final String clientRemoteId, final String jobId, final Configuration applicationConfiguration) { return Configurations.merge(MesosDriverConfiguration.CONF .set(MesosDriverConfiguration.MESOS_MASTER_IP, this.masterIp) .set(MesosDriverConfiguration.JOB_IDENTIFIER, jobId) .set(MesosDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId) .set(MesosDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack) .set(MesosDriverConfiguration.SCHEDULER_DRIVER_CAPACITY, 1) // must be 1 as there is 1 scheduler at the same time .set(MesosDriverConfiguration.RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME) .build(), applicationConfiguration); } }
.set(MultiRuntimeDriverConfiguration.SERIALIZED_RUNTIME_DEFINITION, this.runtimeDefinitionSerializer.toString(runtimeDefinitions)) .build());
/** * Get driver configuration. * * @param jobConf job Configuration to get job id and driver memory. * @return driver configuration. * @throws InjectionException exception while injection. */ private static Configuration getDriverConf(final Configuration jobConf) throws InjectionException { final Injector injector = TANG.newInjector(jobConf); final String jobId = injector.getNamedInstance(JobConf.JobId.class); final int driverMemory = injector.getNamedInstance(JobConf.DriverMemMb.class); return DriverConfiguration.CONF .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(NemoDriver.class)) .set(DriverConfiguration.ON_DRIVER_STARTED, NemoDriver.StartHandler.class) .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, NemoDriver.AllocatedEvaluatorHandler.class) .set(DriverConfiguration.ON_CONTEXT_ACTIVE, NemoDriver.ActiveContextHandler.class) .set(DriverConfiguration.ON_EVALUATOR_FAILED, NemoDriver.FailedEvaluatorHandler.class) .set(DriverConfiguration.ON_CONTEXT_FAILED, NemoDriver.FailedContextHandler.class) .set(DriverConfiguration.ON_DRIVER_STOP, NemoDriver.DriverStopHandler.class) .set(DriverConfiguration.DRIVER_IDENTIFIER, jobId) .set(DriverConfiguration.DRIVER_MEMORY, driverMemory) .build(); }
.set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServicePort) .build()) .build();
return evaluatorConfigurationModule.build();