return ClusterDefaults.builder() .setServices(services) .setProvider(context.<String>deserialize(jsonObj.get(PROVIDER_KEY), String.class))
@Override public JsonElement serialize(ClusterDefaults clusterDefaults, Type type, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); jsonObj.add(SERVICES_KEY, context.serialize(clusterDefaults.getServices())); jsonObj.add(PROVIDER_KEY, context.serialize(clusterDefaults.getProvider())); jsonObj.add(HARDWARE_TYPE_KEY, context.serialize(clusterDefaults.getHardwaretype())); jsonObj.add(IMAGE_TYPE_KEY, context.serialize(clusterDefaults.getImagetype())); jsonObj.add(DNS_SUFFIX_KEY, context.serialize(clusterDefaults.getDnsSuffix())); jsonObj.add(CONFIG_KEY, context.serialize(clusterDefaults.getConfig())); return jsonObj; }
public ClusterDefaults build() { return new ClusterDefaults(services, provider, hardwaretype, imagetype, dnsSuffix, config); } }
private void validateFields(AbstractTemplate to) { Preconditions.checkArgument(to.clusterDefaults.getServices() != null && !to.clusterDefaults.getServices().isEmpty(), "default services must be specified"); Preconditions.checkArgument(to.clusterDefaults.getProvider() != null, "default provider must be specified"); Preconditions.checkArgument(to.clusterDefaults.getDnsSuffix() == null || StringUtils.isValidDNSSuffix(to.clusterDefaults.getDnsSuffix()), to.clusterDefaults.getDnsSuffix() + " is an invalid DNS suffix."); }
if (requiredHardwareType == null || requiredHardwareType.isEmpty()) { requiredHardwareType = template.getClusterDefaults().getHardwaretype(); if (requiredImageType == null || requiredImageType.isEmpty()) { requiredImageType = template.getClusterDefaults().getImagetype(); serviceNames = template.getClusterDefaults().getServices(); dnsSuffix = template.getClusterDefaults().getDnsSuffix();
@Test public void test_07_not_persisted_resolveTemplate() throws Exception { InputStream notPersistedIn = classLoader.getResourceAsStream("partials/cdap-not-persisted.json"); ClusterTemplate notPersisted = gson.fromJson(IOUtils.toString(notPersistedIn), ClusterTemplate.class); ClusterTemplate rt = clusterService.resolveTemplate(account, notPersisted); Assert.assertEquals(notPersisted.getIncludes(), rt.getIncludes()); Assert.assertEquals(notPersisted.getParent(), rt.getParent()); Assert.assertEquals(notPersisted.getName(), rt.getName()); Assert.assertEquals(notPersisted.getDescription(), rt.getDescription()); Assert.assertEquals(sensuPartial.getClusterDefaults().getServices(), rt.getClusterDefaults().getServices()); Assert.assertEquals(sensuPartial.getClusterDefaults().getConfig(), rt.getClusterDefaults().getConfig()); Assert.assertEquals(sensuPartial.getCompatibilities().getServices(), rt.getCompatibilities().getServices()); }
private Set<String> getServices(ClusterTemplate template, ClusterCreateRequest createRequest) { // set cluster service names. Dependency checking is done later on since it involves a lot of potential lookups. Set<String> serviceNames = createRequest.getServices(); if (serviceNames == null || serviceNames.isEmpty()) { serviceNames = template.getClusterDefaults().getServices(); } return serviceNames; }
private Provider getProvider(ClusterTemplate template, ClusterCreateRequest request, EntityStoreView entityStore) throws IOException, MissingEntityException { // make sure the provider exists String providerName = request.getProvider(); if (providerName == null || providerName.isEmpty()) { providerName = template.getClusterDefaults().getProvider(); } Provider provider = entityStore.getProvider(providerName); if (provider == null) { throw new MissingEntityException("provider " + providerName + " does not exist."); } return provider; }
private JsonObject getConfig(ClusterTemplate template, ClusterCreateRequest createRequest) { // use the config from the request if it exists. Otherwise use the template default JsonObject config = createRequest.getConfig(); if (config == null) { config = template.getClusterDefaults().getConfig(); } return config; }
@Test public void test_03_persistentTemplatesConsistence() throws Exception { PartialTemplate ldapInternal = entityStoreView.getPartialTemplate("LDAP-internal"); PartialTemplate sensuInternal = entityStoreView.getPartialTemplate("sensu-internal"); ClusterTemplate cdapDistributedSecureHadoop = entityStoreView.getClusterTemplate("cdap-distributed-secure-hadoop"); ClusterTemplate cdapDistributedInsecure = entityStoreView.getClusterTemplate("cdap-distributed-insecure"); ClusterTemplate cdapDistributed = entityStoreView.getClusterTemplate("cdap-distributed"); Assert.assertNotNull(ldapInternal); Assert.assertNotNull(sensuInternal); Assert.assertNotNull(cdapDistributedSecureHadoop); Assert.assertNotNull(cdapDistributedInsecure); Assert.assertNotNull(cdapDistributed); Assert.assertEquals("Configure Example, Inc. LDAP services", ldapInternal.getDescription()); Assert.assertEquals(true, sensuInternal.isImmutable()); Assert.assertEquals("ldap-internal", ldapInternal.clusterDefaults.getServices().iterator().next()); Assert.assertEquals("ldap-internal", ldapInternal.compatibilities.getServices().iterator().next()); Assert.assertNotNull(ldapInternal.clusterDefaults.getConfig().get("ldap")); Assert.assertEquals("ldap.wrong.com", ldapInternal.clusterDefaults.getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", cdapDistributedSecureHadoop.getDescription()); Assert.assertNotNull(cdapDistributedSecureHadoop.getParent()); Assert.assertEquals("cdap-distributed", cdapDistributedSecureHadoop.getParent().getName()); Assert.assertNotNull(cdapDistributedSecureHadoop.getIncludes()); Assert.assertEquals("LDAP-internal", cdapDistributedSecureHadoop.getIncludes().iterator().next().getName()); Assert.assertEquals(3, cdapDistributedSecureHadoop.clusterDefaults.getServices().size()); Assert.assertNotNull(cdapDistributedSecureHadoop.getClusterDefaults().getConfig().get("hive")); Assert.assertEquals("kerberos-client", cdapDistributedSecureHadoop.getCompatibilities().getServices().iterator().next()); }
@Test public void testGetValidServiceSets() { Set<Set<String>> expected = ImmutableSet.<Set<String>>of( ImmutableSet.of("datanode", "regionserver", "nodemanager"), ImmutableSet.of("namenode", "resourcemanager", "hbasemaster"), ImmutableSet.of("reactor", "zookeeper"), ImmutableSet.of("zookeeper"), ImmutableSet.of("reactor")); Set<String> services = ImmutableSet.of("datanode", "regionserver", "nodemanager", "namenode", "resourcemanager", "hbasemaster", "reactor", "zookeeper"); NodeLayoutGenerator nodeLayoutGenerator = new NodeLayoutGenerator(reactorTemplate, services, ImmutableSet.<String>of(), ImmutableSet.<String>of()); Assert.assertEquals( expected, nodeLayoutGenerator.findValidServiceSets(reactorTemplate.getClusterDefaults().getServices())); }
@Test public void testInvalidNumMachines() throws Exception { // when its below the min ClusterCreateRequest clusterCreateRequest =ClusterCreateRequest.builder() .setName("my-cluster") .setClusterTemplateName(reactorTemplate.getName()) .setProviderName(reactorTemplate.getClusterDefaults().getProvider()) .setNumMachines(1) .build(); assertResponseStatus(doPostExternalAPI("/clusters", gson.toJson(clusterCreateRequest), USER1_HEADERS), HttpResponseStatus.BAD_REQUEST); // when its above the max clusterCreateRequest = ClusterCreateRequest.builder() .setName("my-cluster") .setClusterTemplateName(reactorTemplate.getName()) .setProviderName(reactorTemplate.getClusterDefaults().getProvider()) .setNumMachines(500) .build(); assertResponseStatus(doPostExternalAPI("/clusters", gson.toJson(clusterCreateRequest), USER1_HEADERS), HttpResponseStatus.BAD_REQUEST); }
@Test public void test_06_resolveTemplateTest() throws Exception { ClusterTemplate rt = clusterService.resolveTemplate(account, secureTemplate); Assert.assertEquals(secureTemplate.getIncludes(), rt.getIncludes()); Assert.assertEquals(secureTemplate.getParent(), rt.getParent()); Assert.assertEquals(secureTemplate.getName(), rt.getName()); Assert.assertEquals(secureTemplate.getDescription(), rt.getDescription()); Assert.assertEquals(defaultsServices, rt.getClusterDefaults().getServices()); Assert.assertEquals(16, rt.getClusterDefaults().getConfig().entrySet().size()); Assert.assertEquals("rackspace", rt.getClusterDefaults().getProvider()); Assert.assertEquals("standard-large", rt.getClusterDefaults().getHardwaretype()); Assert.assertEquals("centos6", rt.getClusterDefaults().getImagetype()); Assert.assertEquals("example.com", rt.getClusterDefaults().getDnsSuffix()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustCoexist().size()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustNotCoexist().size()); Assert.assertEquals(serviceConstraints, rt.getConstraints().getServiceConstraints()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getInitial()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getStep()); Assert.assertEquals(compatibilitiesHardwaretypes, rt.getCompatibilities().getHardwaretypes()); Assert.assertEquals(compatibilitiesImagetypes, rt.getCompatibilities().getImagetypes()); Assert.assertEquals(compatibilitiesServices, rt.getCompatibilities().getServices()); //overrides in child Assert.assertEquals("ldap.correct.com", rt.getClusterDefaults().getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertNull(rt.getLabel()); Assert.assertNull(rt.getIcon()); Assert.assertEquals("cdap-distributed-secure-hadoop", rt.getName()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", rt.getDescription()); }
@Test public void testNodesMustHaveServices() throws Exception { Set<String> services = ImmutableSet.of("svc1", "svc2", "svc3"); ClusterTemplate template = ClusterTemplate.builder() .setName("simple") .setDescription("all services on all nodes template") .setClusterDefaults(ClusterDefaults.builder().setServices(services).setProvider("joyent").build()) .setCompatibilities(Compatibilities.builder().setServices(services).build()) .setConstraints(new Constraints( ImmutableMap.<String, ServiceConstraint>of("svc1", new ServiceConstraint(null, null, 1, 1)), new LayoutConstraint( ImmutableSet.<Set<String>>of(ImmutableSet.of("svc1", "svc2", "svc3")), ImmutableSet.<Set<String>>of() ), SizeConstraint.EMPTY )).build(); List<NodeLayout> expected = ImmutableList.of( new NodeLayout("small", "centos6", ImmutableSet.of("svc1", "svc2", "svc3")) ); NodeLayoutGenerator nodeLayoutGenerator = new NodeLayoutGenerator(template, services, ImmutableSet.<String>of("small"), ImmutableSet.<String>of("centos6")); List<NodeLayout> actual = nodeLayoutGenerator.generateNodeLayoutPreferences(); Assert.assertEquals(expected, actual); }
@Test public void testGetUnconstrainedService() throws Exception { NodeLayoutGenerator nodeLayoutGenerator = new NodeLayoutGenerator(reactorTemplate2, reactorTemplate2.getClusterDefaults().getServices(), ImmutableSet.<String>of(), ImmutableSet.<String>of()); Set<String> unconstrained = nodeLayoutGenerator.findUnconstrainedServices(); Assert.assertEquals(ImmutableSet.of("hosts", "firewall"), unconstrained); }
@Test public void testNoSolutionReturnsNull() { Set<String> services = ImmutableSet.of("svc1", "svc2", "svc3"); ClusterTemplate template = ClusterTemplate.builder() .setName("simple") .setDescription("all services on all nodes template") .setClusterDefaults(ClusterDefaults.builder().setServices(services).setProvider("joyent").build()) .setCompatibilities(Compatibilities.builder().setServices(services).build()) .setConstraints(new Constraints( ImmutableMap.<String, ServiceConstraint>of("svc1", new ServiceConstraint(null, null, 1, 1)), new LayoutConstraint( ImmutableSet.<Set<String>>of(ImmutableSet.of("svc1", "svc2", "svc3")), ImmutableSet.<Set<String>>of() ), SizeConstraint.EMPTY)) .build(); List<NodeLayout> nodePreferences = ImmutableList.of( new NodeLayout("small", "centos6", ImmutableSet.of("svc1", "svc2", "svc3")) ); ClusterLayoutFinder finder = new ClusterLayoutFinder(nodePreferences, template, services, 2); Assert.assertNull(finder.findValidNodeCounts()); }
private void assertClusterLayout(List<NodeLayout> nodeLayouts, ClusterTemplate template, int[] nodeCounts, int numMachines, boolean expected) { ClusterLayoutFinder clusterLayoutFinder = new ClusterLayoutFinder(nodeLayouts, template, template.getClusterDefaults().getServices(), numMachines); Assert.assertEquals(expected, clusterLayoutFinder.isValidCluster(nodeCounts)); } }
private void copyMainProps(AbstractTemplate from, AbstractTemplate to, Set<String> immutables, boolean isImmutable) throws TemplateImmutabilityException { //merge defaults if(!from.clusterDefaults.equals(ClusterDefaults.EMPTY_CLUSTER_DEFAULTS)){ to.clusterDefaults = ClusterDefaults.builder() .setServices(mergeSet(to.clusterDefaults.services, from.clusterDefaults.services)) .setConfig(mergeConfig(to.clusterDefaults.config, from.clusterDefaults.config, immutables, isImmutable)) .setDNSSuffix(mergeString(to.clusterDefaults.dnsSuffix, from.clusterDefaults.dnsSuffix)) .setHardwaretype(mergeString(to.clusterDefaults.hardwaretype, from.clusterDefaults.hardwaretype)) .setImagetype(mergeString(to.clusterDefaults.imagetype, from.clusterDefaults.imagetype)) .setProvider(mergeString(to.clusterDefaults.provider, from.clusterDefaults.provider)) .build(); } //merge compatibilities services if(!from.compatibilities.equals(Compatibilities.EMPTY_COMPATIBILITIES)) { to.compatibilities = Compatibilities.builder() .setServices(mergeSet(to.compatibilities.services, from.compatibilities.services)) .setHardwaretypes(mergeSet(to.compatibilities.hardwaretypes, from.compatibilities.hardwaretypes)) .setImagetypes(mergeSet(to.compatibilities.imagetypes, from.compatibilities.imagetypes)) .build(); } }
@Test public void testGetValidNodeLayouts() { Set<NodeLayout> expected = ImmutableSet.of( new NodeLayout("large-mem", "centos6", ImmutableSet.of("namenode", "resourcemanager", "hbasemaster")), new NodeLayout("large-mem", "ubuntu12", ImmutableSet.of("namenode", "resourcemanager", "hbasemaster")), new NodeLayout("medium", "centos6", ImmutableSet.of("datanode", "nodemanager", "regionserver")), new NodeLayout("medium", "ubuntu12", ImmutableSet.of("datanode", "nodemanager", "regionserver")), new NodeLayout("large-cpu", "centos6", ImmutableSet.of("datanode", "nodemanager", "regionserver")), new NodeLayout("large-cpu", "ubuntu12", ImmutableSet.of("datanode", "nodemanager", "regionserver")), new NodeLayout("medium", "centos6", ImmutableSet.of("reactor", "zookeeper")), new NodeLayout("small", "centos6", ImmutableSet.of("zookeeper")), new NodeLayout("medium", "centos6", ImmutableSet.of("zookeeper")), new NodeLayout("medium", "centos6", ImmutableSet.of("reactor")), new NodeLayout("large", "centos6", ImmutableSet.of("reactor")), new NodeLayout("medium", "ubuntu12", ImmutableSet.of("reactor")), new NodeLayout("large", "ubuntu12", ImmutableSet.of("reactor")), new NodeLayout("medium", "sl6", ImmutableSet.of("reactor")), new NodeLayout("large", "sl6", ImmutableSet.of("reactor")) ); Set<String> services = ImmutableSet.of("datanode", "regionserver", "nodemanager", "namenode", "resourcemanager", "hbasemaster", "reactor", "zookeeper"); NodeLayoutGenerator nodeLayoutGenerator = new NodeLayoutGenerator(reactorTemplate, services, ImmutableSet.<String>of("small", "medium", "large", "large-cpu", "large-mem"), ImmutableSet.<String>of("centos6", "ubuntu12", "sl6")); Set<Set<String>> validServiceSets = nodeLayoutGenerator.findValidServiceSets(reactorTemplate.getClusterDefaults().getServices()); Assert.assertEquals(expected, nodeLayoutGenerator.findValidNodeLayouts(validServiceSets)); }
@BeforeClass public static void initData() throws Exception { JsonObject defaultClusterConfig = new JsonObject(); defaultClusterConfig.addProperty("defaultconfig", "value1"); smallTemplate = ClusterTemplate.builder() .setName("one-machine") .setClusterDefaults( ClusterDefaults.builder() .setServices("zookeeper") .setProvider("rackspace") .setConfig(defaultClusterConfig) .build()) .setCompatibilities(Compatibilities.builder().setServices("zookeeper").build()) .setAdministration(new Administration(LeaseDuration.of("10s", "30s", "5s"))) .build(); }