for (String component : instanceDefinition.getResourceOperations().getComponentNames()) { instanceDefinition.getAppConfOperations().getOrAddComponent(component); instanceDefinition.resolve(); ConfTreeOperations resources = instanceDefinition.getResourceOperations(); resourcesSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getResources()); appConfSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getAppConf()); internalsSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getInternal()); instanceDefinitionSnapshot = new AggregateConf(resourcesSnapshot.confTree, appConfSnapshot.confTree, internalsSnapshot.confTree); instanceDefinitionSnapshot.setName(instanceDefinition.getName());
/** * Build the image dir. This path is relative and only valid at the far end * @param instance instance options * @param bindir bin subdir * @param script script in bin subdir * @return the path to the script * @throws FileNotFoundException if a file is not found, or it is not a directory* */ public String buildPathToScript(AggregateConf instance, String bindir, String script) throws FileNotFoundException { return buildPathToScript(instance.getInternalOperations(), bindir, script); } /**
/** * Take a snapshot of the configuration * @param instanceDefinition source * @throws IOException marshalling/copying problems */ public AggregateConf(AggregateConf instanceDefinition) throws IOException { ConfTreeOperations resourcesSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getResources()); ConfTreeOperations appConfSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getAppConf()); ConfTreeOperations internalsSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getInternal()); //build a new aggregate from the snapshots setResources(resourcesSnapshot.confTree); setAppConf(appConfSnapshot.confTree); setInternal(internalsSnapshot.confTree); }
public AggregateConf(ConfTree resources, ConfTree appConf, ConfTree internal) { setResources(resources); setAppConf(appConf); setInternal(internal); }
private void saveConf(AggregateConf conf) throws IOException { confTreeSerDeser.save(fileSystem, internal, conf.getInternal(), true); confTreeSerDeser.save(fileSystem, resources, conf.getResources(), true); confTreeSerDeser.save(fileSystem, app_conf, conf.getAppConf(), true); }
private AggregateConf prepareConfForAgentStateTests() { ConfTree tree = new ConfTree(); tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); AggregateConf instanceDefinition = new AggregateConf(); instanceDefinition.setInternal(tree); instanceDefinition.setAppConf(tree); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.APP_DEF, "."); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.AGENT_CONF, "."); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.AGENT_VERSION, "."); instanceDefinition.getResourceOperations().getOrAddComponent( "HBASE_MASTER"); instanceDefinition.getResourceOperations().getOrAddComponent( "HBASE_REGIONSERVER"); return instanceDefinition; }
instanceDefinition.resolve(); launchedInstanceDefinition = instanceDefinition; ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); MapOperations internalOptions = internalOperations.getGlobalOptions(); ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); Path generatedConfDirPath = createPathThatMustExist(internalOptions.getMandatoryOption( internalOptions.getMandatoryOption(INTERNAL_PROVIDER_NAME)); if (log.isDebugEnabled()) { log.debug(instanceDefinition.toString());
instanceDefinition.setName(clustername); AggregateConf resolvedInstance = new AggregateConf( instanceDefinition); resolvedInstance.resolve(); .getAppConfOperations().getComponent(COMPONENT_AM); instanceDefinition.getAppConfOperations().set(KEY_SECURITY_ENABLED, securityEnabled); MapOperations component = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM); certificateManager.initialize(component, appMasterHostname, appMasterContainerID.toString(), clustername); certificateManager.setPassphrase(instanceDefinition.getPassphrase()); instanceDefinition.getName(), appState.getRolePriorityMap()); initCompleted.set(true); scheduleFailureWindowResets(instanceDefinition.getResources()); scheduleEscalation(instanceDefinition.getInternal()); scheduleHealthThresholdMonitor(instanceDefinition.getResources());
aggregateConf.resolve(); aggregateConf.getInternal().global); SliderUtils.mergeMapsIgnoreDuplicateKeys(options, aggregateConf.getAppConf().global); SliderUtils.mergeMapsIgnoreDuplicateKeys(options, aggregateConf.getResources().global); mergeInComponentMap(cd, aggregateConf.getInternal()); mergeInComponentMap(cd, aggregateConf.getAppConf()); mergeInComponentMap(cd, aggregateConf.getResources()); aggregateConf.getInternalOperations().getGlobalOptions(); MapOperations appOptions = aggregateConf.getAppConfOperations().getGlobalOptions();
public void resolve() throws BadConfigException, IOException, BadClusterStateException { ConfTreeOperations appConf = instanceDescription.getAppConfOperations(); ConfTreeOperations resources = instanceDescription.getResourceOperations(); " doesn't exist"); AggregateConf componentConf = new AggregateConf(); ConfPersister persister = new ConfPersister(coreFS, componentClusterDir); try { ConfTreeOperations componentAppConf = componentConf.getAppConfOperations(); String externalAppDef = componentAppConf.get(AgentKeys.APP_DEF); if (SliderUtils.isSet(externalAppDef)) { for (String rcomp : componentConf.getResourceOperations() .getComponentNames()) { if (COMPONENT_AM.equals(rcomp)) { componentAppConf.getOrAddComponent(rcomp); componentConf.resolve(); for (String rcomp : componentConf.getResourceOperations() .getComponentNames()) { if (COMPONENT_AM.equals(rcomp)) { mergeExternalComponent(resources, componentConf.getResourceOperations(), component, getNextPriority());
/** * Propagate any critical principals from the current site config down to the HBase one. */ public void propagatePrincipals() { String dfsPrincipal = conf.get(SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); if (dfsPrincipal != null) { String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX + SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal); } }
ConfTreeOperations template = ConfTreeOperations.fromResource(internalTemplate); instanceConf.getInternalOperations() .mergeWithoutOverwrite(template.confTree); ConfTreeOperations resTemplate = ConfTreeOperations.fromResource(resourceTemplate); instanceConf.getResourceOperations() .mergeWithoutOverwrite(resTemplate.confTree); ConfTreeOperations template = ConfTreeOperations.fromResource(appConfTemplate); instanceConf.getAppConfOperations() .mergeWithoutOverwrite(template.confTree);
@Test public void testContainerKeystoreGenerationViaStoresGeneratorOverrideGlobalSetting() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = setupComponentOptions(true, null, "app1.component1.password.property", null, null); instanceDefinition.getAppConf().components.put("component1", compOps); instanceDefinition.getAppConf().global.put( "app1.component1.password.property", "password"); instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "false"); instanceDefinition.resolve(); SecurityStore[] files = StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); assertEquals("wrong number of stores", 1, files.length); validateKeystore(files[0].getFile(), "testhost", "cahost"); }
/** * Start an asychronous launch operation * @param assignment container assignment * @param clusterSpec cluster spec to use for template * @param credentials credentials to use */ public void launchRole(ContainerAssignment assignment, AggregateConf clusterSpec, Credentials credentials) { RoleStatus role = assignment.role; String roleName = role.getName(); String roleGroup = role.getGroup(); // prelaunch safety check Preconditions.checkArgument(provider.isSupportedRole(roleName)); RoleLaunchService.RoleLauncher launcher = new RoleLaunchService.RoleLauncher(assignment, clusterSpec, clusterSpec.getResourceOperations().getOrAddComponent(roleGroup), clusterSpec.getAppConfOperations().getOrAddComponent(roleGroup), credentials); execute(launcher); }
/** * Get the node failure threshold for a specific role, falling back to * the global one if not * @param roleGroup role group * @return the threshold for failures */ private int getNodeFailureThresholdForRole(String roleGroup) { ConfTreeOperations resources = instanceDefinition.getResourceOperations(); return resources.getComponentOptInt(roleGroup, NODE_FAILURE_THRESHOLD, nodeFailureThreshold); }
/** * Build the accumulo-site.xml file * This the configuration used by Accumulo directly * @param instanceDescription this is the cluster specification used to define this * @return a map of the dynamic bindings for this Slider instance */ public Map<String, String> buildSiteConfFromInstance( AggregateConf instanceDescription) throws BadConfigException { ConfTreeOperations appconf = instanceDescription.getAppConfOperations(); MapOperations globalAppOptions = appconf.getGlobalOptions(); MapOperations globalInstanceOptions = instanceDescription.getInternalOperations().getGlobalOptions(); Map<String, String> sitexml = new HashMap<String, String>(); providerUtils.propagateSiteOptions(globalAppOptions, sitexml); propagateClientFSBinding(sitexml); setDatabasePath(sitexml, globalInstanceOptions.getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH)); String quorum = globalAppOptions.getMandatoryOption(OptionKeys.ZOOKEEPER_QUORUM); sitexml.put(AccumuloConfigFileOptions.ZOOKEEPER_HOST, quorum); return sitexml; }
@Test public void testParameterParsing() throws IOException { AgentProviderService aps = createAgentProviderService(new Configuration()); AggregateConf aggConf = new AggregateConf(); ConfTreeOperations treeOps = aggConf.getAppConfOperations(); treeOps.getGlobalOptions().put(AgentKeys.SYSTEM_CONFIGS, "core-site,yarn-site, core-site "); List<String> configs = aps.getSystemConfigurationsRequested(treeOps); Assert.assertEquals(2, configs.size()); Assert.assertTrue(configs.contains("core-site")); Assert.assertFalse(configs.contains("bore-site")); }
protected AggregateConf getAggregateConf() { try { JsonSerDeser<ConfTree> confTreeJsonSerDeser = new JsonSerDeser<>(ConfTree.class); AggregateConf aggregateConf = new AggregateConf( confTreeJsonSerDeser.fromResource(EXAMPLES + "resources.json"), confTreeJsonSerDeser.fromResource(EXAMPLES + "app_configuration.json"), confTreeJsonSerDeser.fromResource(EXAMPLES + "internal.json") ); aggregateConf.setName("test"); return aggregateConf; } catch (IOException e) { throw new AssertionError(e.getMessage(), e); } } }
@Test public void testPrepareAMAndConfigForLaunch() throws Exception { AgentClientProvider provider = new AgentClientProvider(null); SliderFileSystem sfs = PowerMock.createMock(SliderFileSystem.class); FileSystem fs = PowerMock.createMock(FileSystem.class); Configuration serviceConf = PowerMock.createMock(Configuration.class); PowerMock.mockStatic(ProviderUtils.class); expect(sfs.getFileSystem()).andReturn(fs); expect(fs.mkdirs(anyObject(Path.class))).andReturn(true); expect(ProviderUtils.addAgentTar( anyObject(), anyObject(String.class), anyObject(SliderFileSystem.class), anyObject(Path.class))). andReturn(true); AggregateConf instanceDefinition = new AggregateConf(); ConfTree tree = new ConfTree(); tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); instanceDefinition.setInternal(tree); PowerMock.replay(sfs, fs, serviceConf, ProviderUtils.class); provider.prepareAMAndConfigForLaunch( sfs, serviceConf, null, instanceDefinition, null, null, null, null, null, false); Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH)); tree.global.remove(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); // Verify that slider-agent.tar.gz got added Path tempPath = new Path(".", "temp"); provider.prepareAMAndConfigForLaunch( sfs, serviceConf, null, instanceDefinition, null, null, null, null, tempPath, false); PowerMock.verify(sfs, fs, ProviderUtils.class); Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH)); }
/** * Load the persistent cluster description * @param clustername name of the cluster * @return the description in the filesystem * @throws IOException any problems loading -including a missing file */ @VisibleForTesting public AggregateConf loadPersistedClusterDescription(String clustername) throws IOException, SliderException, LockAcquireFailedException { Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); ConfPersister persister = new ConfPersister(sliderFileSystem, clusterDirectory); AggregateConf instanceDescription = new AggregateConf(); persister.load(instanceDescription); return instanceDescription; }