/** * Load the persistent cluster description * @param clustername name of the cluster * @return the description in the filesystem * @throws IOException any problems loading -including a missing file */ @VisibleForTesting public AggregateConf loadPersistedClusterDescription(String clustername) throws IOException, SliderException, LockAcquireFailedException { Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); ConfPersister persister = new ConfPersister(sliderFileSystem, clusterDirectory); AggregateConf instanceDescription = new AggregateConf(); persister.load(instanceDescription); return instanceDescription; }
/** * Load the persistent cluster description * @param clustername name of the cluster * @return the description in the filesystem * @throws IOException any problems loading -including a missing file */ @VisibleForTesting public AggregateConf loadPersistedClusterDescription(String clustername) throws IOException, SliderException, LockAcquireFailedException { Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); ConfPersister persister = new ConfPersister(sliderFileSystem, clusterDirectory); AggregateConf instanceDescription = new AggregateConf(); persister.load(instanceDescription); return instanceDescription; }
protected AggregateConf getAggregateConf() { try { JsonSerDeser<ConfTree> confTreeJsonSerDeser = new JsonSerDeser<>(ConfTree.class); AggregateConf aggregateConf = new AggregateConf( confTreeJsonSerDeser.fromResource(EXAMPLES + "resources.json"), confTreeJsonSerDeser.fromResource(EXAMPLES + "app_configuration.json"), confTreeJsonSerDeser.fromResource(EXAMPLES + "internal.json") ); aggregateConf.setName("test"); return aggregateConf; } catch (IOException e) { throw new AssertionError(e.getMessage(), e); } } }
protected AggregateConf getAggregateConf() { try { JsonSerDeser<ConfTree> confTreeJsonSerDeser = new JsonSerDeser<>(ConfTree.class); AggregateConf aggregateConf = new AggregateConf( confTreeJsonSerDeser.fromResource(EXAMPLES + "resources.json"), confTreeJsonSerDeser.fromResource(EXAMPLES + "app_configuration.json"), confTreeJsonSerDeser.fromResource(EXAMPLES + "internal.json") ); aggregateConf.setName("test"); return aggregateConf; } catch (IOException e) { throw new AssertionError(e.getMessage(), e); } } }
@Test public void testParameterParsing() throws IOException { AgentProviderService aps = createAgentProviderService(new Configuration()); AggregateConf aggConf = new AggregateConf(); ConfTreeOperations treeOps = aggConf.getAppConfOperations(); treeOps.getGlobalOptions().put(AgentKeys.SYSTEM_CONFIGS, "core-site,yarn-site, core-site "); List<String> configs = aps.getSystemConfigurationsRequested(treeOps); Assert.assertEquals(2, configs.size()); Assert.assertTrue(configs.contains("core-site")); Assert.assertFalse(configs.contains("bore-site")); }
@Test public void testParameterParsing() throws IOException { AgentProviderService aps = createAgentProviderService(new Configuration()); AggregateConf aggConf = new AggregateConf(); ConfTreeOperations treeOps = aggConf.getAppConfOperations(); treeOps.getGlobalOptions().put(AgentKeys.SYSTEM_CONFIGS, "core-site,yarn-site, core-site "); List<String> configs = aps.getSystemConfigurationsRequested(treeOps); Assert.assertEquals(2, configs.size()); Assert.assertTrue(configs.contains("core-site")); Assert.assertFalse(configs.contains("bore-site")); }
@Test public void testValidateInstanceDefinition() throws Exception { AgentClientProvider provider = new AgentClientProvider(null); AggregateConf instanceDefinition = new AggregateConf(); try { provider.validateInstanceDefinition(instanceDefinition, null); Assert.assertFalse("Should fail with BadConfigException", true); } catch (BadConfigException e) { log.info(e.toString()); Assert.assertTrue(e.getMessage().contains("Application definition must be provided")); } } }
/** * Implementation of cluster flexing. * It should be the only way that anything -even the AM itself on startup- * asks for nodes. * @param resources the resource tree * @throws SliderException slider problems, including invalid configs * @throws IOException IO problems */ public void flexCluster(ConfTree resources) throws IOException, SliderException { AggregateConf newConf = new AggregateConf(appState.getInstanceDefinitionSnapshot()); newConf.setResources(resources); // verify the new definition is valid sliderAMProvider.validateInstanceDefinition(newConf); providerService.validateInstanceDefinition(newConf); appState.updateResourceDefinitions(resources); // reset the scheduled windows...the values // may have changed appState.resetFailureCounts(); // ask for more containers if needed reviewRequestAndReleaseNodes("flexCluster"); }
@Test public void testValidateInstanceDefinition() throws Exception { AgentClientProvider provider = new AgentClientProvider(null); AggregateConf instanceDefinition = new AggregateConf(); try { provider.validateInstanceDefinition(instanceDefinition, null); Assert.assertFalse("Should fail with BadConfigException", true); } catch (BadConfigException e) { log.info(e.toString()); Assert.assertTrue(e.getMessage().contains("Application definition must be provided")); } } }
@Test public void testContainerKeystoreGenerationViaStoresGeneratorOverrideGlobalSetting() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = setupComponentOptions(true, null, "app1.component1.password.property", null, null); instanceDefinition.getAppConf().components.put("component1", compOps); instanceDefinition.getAppConf().global.put( "app1.component1.password.property", "password"); instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "false"); instanceDefinition.resolve(); SecurityStore[] files = StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); assertEquals("wrong number of stores", 1, files.length); validateKeystore(files[0].getFile(), "testhost", "cahost"); }
@Test public void testContainerKeystoreGenerationViaStoresGeneratorOverrideGlobalSetting() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = setupComponentOptions(true, null, "app1.component1.password.property", null, null); instanceDefinition.getAppConf().components.put("component1", compOps); instanceDefinition.getAppConf().global.put( "app1.component1.password.property", "password"); instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "false"); instanceDefinition.resolve(); SecurityStore[] files = StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); assertEquals("wrong number of stores", 1, files.length); validateKeystore(files[0].getFile(), "testhost", "cahost"); }
/** * Get the AM instance definition. * <p> * See {@link SliderClusterProtocol#getInstanceDefinition(Messages.GetInstanceDefinitionRequestProto)} * @return current slider AM aggregate definition * @throws YarnException * @throws IOException */ public AggregateConf getInstanceDefinition() throws YarnException, IOException { Messages.GetInstanceDefinitionRequestProto.Builder builder = Messages.GetInstanceDefinitionRequestProto.newBuilder(); Messages.GetInstanceDefinitionRequestProto request = builder.build(); Messages.GetInstanceDefinitionResponseProto response = appMaster.getInstanceDefinition(request); ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); ConfTree internal = confTreeSerDeser.fromJson(response.getInternal()); ConfTree resources = confTreeSerDeser.fromJson(response.getResources()); ConfTree app = confTreeSerDeser.fromJson(response.getApplication()); AggregateConf instanceDefinition = new AggregateConf(resources, app, internal); return instanceDefinition; } /**
@Test public void testContainerStoresGenerationMisconfiguration() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = new MapOperations(); compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); setupCredentials(instanceDefinition, "cant.be.found", null); try { StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); Assert.fail("SliderException should have been generated"); } catch (SliderException e) { // ignore - should be thrown } }
@Test public void testContainerStoresGenerationMisconfiguration() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = new MapOperations(); compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); setupCredentials(instanceDefinition, "cant.be.found", null); try { StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); Assert.fail("SliderException should have been generated"); } catch (SliderException e) { // ignore - should be thrown } }
@Test public void testContainerKeystoreGenerationViaStoresGenerator() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = new MapOperations(); instanceDefinition.getAppConf().components.put("component1", compOps); compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY, "app1.component1.password.property"); compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); instanceDefinition.getAppConf().global.put( "app1.component1.password.property", "password"); instanceDefinition.resolve(); SecurityStore[] files = StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); assertEquals("wrong number of stores", 1, files.length); validateKeystore(files[0].getFile(), "testhost", "cahost"); }
@Test public void testContainerKeystoreGenerationViaStoresGeneratorUsingGlobalProps() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = new MapOperations(); instanceDefinition.getAppConf().components.put("component1", compOps); compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY, "app1.component1.password.property"); instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); compOps.put( "app1.component1.password.property", "password"); instanceDefinition.resolve(); SecurityStore[] files = StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); assertEquals("wrong number of stores", 1, files.length); validateKeystore(files[0].getFile(), "testhost", "cahost"); }
@Test public void testContainerKeystoreGenerationViaStoresGenerator() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = new MapOperations(); instanceDefinition.getAppConf().components.put("component1", compOps); compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY, "app1.component1.password.property"); compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); instanceDefinition.getAppConf().global.put( "app1.component1.password.property", "password"); instanceDefinition.resolve(); SecurityStore[] files = StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); assertEquals("wrong number of stores", 1, files.length); validateKeystore(files[0].getFile(), "testhost", "cahost"); }
@Test public void testContainerKeystoreGenerationViaStoresGeneratorUsingGlobalProps() throws Exception { AggregateConf instanceDefinition = new AggregateConf(); MapOperations compOps = new MapOperations(); instanceDefinition.getAppConf().components.put("component1", compOps); compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY, "app1.component1.password.property"); instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); compOps.put( "app1.component1.password.property", "password"); instanceDefinition.resolve(); SecurityStore[] files = StoresGenerator.generateSecurityStores("testhost", "container1", "component1", instanceDefinition, compOps); assertEquals("wrong number of stores", 1, files.length); validateKeystore(files[0].getFile(), "testhost", "cahost"); }
private AggregateConf prepareConfForAgentStateTests() { ConfTree tree = new ConfTree(); tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); AggregateConf instanceDefinition = new AggregateConf(); instanceDefinition.setInternal(tree); instanceDefinition.setAppConf(tree); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.APP_DEF, "."); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.AGENT_CONF, "."); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.AGENT_VERSION, "."); instanceDefinition.getResourceOperations().getOrAddComponent( "HBASE_MASTER"); instanceDefinition.getResourceOperations().getOrAddComponent( "HBASE_REGIONSERVER"); return instanceDefinition; }
private AggregateConf prepareConfForAgentStateTests() { ConfTree tree = new ConfTree(); tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); AggregateConf instanceDefinition = new AggregateConf(); instanceDefinition.setInternal(tree); instanceDefinition.setAppConf(tree); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.APP_DEF, "."); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.AGENT_CONF, "."); instanceDefinition.getAppConfOperations().getGlobalOptions() .put(AgentKeys.AGENT_VERSION, "."); instanceDefinition.getResourceOperations().getOrAddComponent( "HBASE_MASTER"); instanceDefinition.getResourceOperations().getOrAddComponent( "HBASE_REGIONSERVER"); return instanceDefinition; }