@Override public Compatibilities deserialize(JsonElement json, Type type, JsonDeserializationContext context) throws JsonParseException { JsonObject jsonObj = json.getAsJsonObject(); Set<String> hardwaretypes = jsonObj.get("hardwaretypes") != null ? context.<Set<String>>deserialize(jsonObj.get("hardwaretypes"), typeToken) : ImmutableSet.<String>of(); Set<String> imagetypes = jsonObj.get("imagetypes") != null ? context.<Set<String>>deserialize(jsonObj.get("imagetypes"), typeToken) : ImmutableSet.<String>of(); Set<String> services = jsonObj.get("services") != null ? context.<Set<String>>deserialize(jsonObj.get("services"), typeToken) : ImmutableSet.<String>of(); return Compatibilities.builder() .setHardwaretypes(hardwaretypes) .setImagetypes(imagetypes) .setServices(services) .build(); } }
@Override public JsonElement serialize(Compatibilities compatibilities, Type type, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); jsonObj.add("hardwaretypes", context.serialize(compatibilities.getHardwaretypes())); jsonObj.add("imagetypes", context.serialize(compatibilities.getHardwaretypes())); jsonObj.add("services", context.serialize(compatibilities.getHardwaretypes())); return jsonObj; }
private void validateServiceCompatibilities(Compatibilities compatibilities, Set<String> services) { if (!compatibilities.compatibleWithServices(services)) { Set<String> incompatibleServices = Sets.difference(services, compatibilities.getServices()); if (!incompatibleServices.isEmpty()) { String incompatibleStr = Joiner.on(',').join(incompatibleServices); throw new IllegalArgumentException(incompatibleStr + " are incompatible with the cluster"); } } }
public boolean isCompatibleWithTemplate(ClusterTemplate template) { // check all services are compatible Compatibilities compatibilities = template.getCompatibilities(); if (!compatibilities.compatibleWithServices(serviceCounts.elementSet())) { return false; } // check all image types and hardware types are compatible Set<String> imageTypes = Sets.newHashSet(); Set<String> hardwareTypes = Sets.newHashSet(); for (NodeLayout nodeLayout : layout.elementSet()) { imageTypes.add(nodeLayout.getImageTypeName()); hardwareTypes.add(nodeLayout.getHardwareTypeName()); } // check all hardware types are compatible if (!compatibilities.compatibleWithImageTypes(imageTypes) || !compatibilities.compatibleWithHardwareTypes(hardwareTypes)) { return false; } return satisfiesConstraints(template.getConstraints()); }
Compatibilities newCompatibilities = Compatibilities.builder() .setHardwaretypes(template.getCompatibilities().getHardwaretypes()) .setImagetypes(template.getCompatibilities().getImagetypes()) .setServices("namenode", "datanode", "zookeeper") .build(); newCompatibilities = Compatibilities.builder() .setHardwaretypes("large") .setImagetypes(template.getCompatibilities().getImagetypes()) .setServices(template.getCompatibilities().getServices()) .build(); template = copyWithNewCompatibilities(template, newCompatibilities); newCompatibilities = Compatibilities.builder() .setHardwaretypes(template.getCompatibilities().getHardwaretypes()) .setImagetypes("ubuntu12") .setServices(template.getCompatibilities().getServices()) .build(); template = copyWithNewCompatibilities(template, newCompatibilities);
Compatibilities.builder() .setHardwaretypes(template.getCompatibilities().getHardwaretypes()) .setImagetypes(newCompatibleImages) .setServices(template.getCompatibilities().getServices()) .build(); ClusterTemplate updatedTemplate = ClusterTemplate.builder()
@Test public void test_06_resolveTemplateTest() throws Exception { ClusterTemplate rt = clusterService.resolveTemplate(account, secureTemplate); Assert.assertEquals(secureTemplate.getIncludes(), rt.getIncludes()); Assert.assertEquals(secureTemplate.getParent(), rt.getParent()); Assert.assertEquals(secureTemplate.getName(), rt.getName()); Assert.assertEquals(secureTemplate.getDescription(), rt.getDescription()); Assert.assertEquals(defaultsServices, rt.getClusterDefaults().getServices()); Assert.assertEquals(16, rt.getClusterDefaults().getConfig().entrySet().size()); Assert.assertEquals("rackspace", rt.getClusterDefaults().getProvider()); Assert.assertEquals("standard-large", rt.getClusterDefaults().getHardwaretype()); Assert.assertEquals("centos6", rt.getClusterDefaults().getImagetype()); Assert.assertEquals("example.com", rt.getClusterDefaults().getDnsSuffix()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustCoexist().size()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustNotCoexist().size()); Assert.assertEquals(serviceConstraints, rt.getConstraints().getServiceConstraints()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getInitial()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getStep()); Assert.assertEquals(compatibilitiesHardwaretypes, rt.getCompatibilities().getHardwaretypes()); Assert.assertEquals(compatibilitiesImagetypes, rt.getCompatibilities().getImagetypes()); Assert.assertEquals(compatibilitiesServices, rt.getCompatibilities().getServices()); //overrides in child Assert.assertEquals("ldap.correct.com", rt.getClusterDefaults().getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertNull(rt.getLabel()); Assert.assertNull(rt.getIcon()); Assert.assertEquals("cdap-distributed-secure-hadoop", rt.getName()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", rt.getDescription()); }
private void copyMainProps(AbstractTemplate from, AbstractTemplate to, Set<String> immutables, boolean isImmutable) throws TemplateImmutabilityException { //merge defaults if(!from.clusterDefaults.equals(ClusterDefaults.EMPTY_CLUSTER_DEFAULTS)){ to.clusterDefaults = ClusterDefaults.builder() .setServices(mergeSet(to.clusterDefaults.services, from.clusterDefaults.services)) .setConfig(mergeConfig(to.clusterDefaults.config, from.clusterDefaults.config, immutables, isImmutable)) .setDNSSuffix(mergeString(to.clusterDefaults.dnsSuffix, from.clusterDefaults.dnsSuffix)) .setHardwaretype(mergeString(to.clusterDefaults.hardwaretype, from.clusterDefaults.hardwaretype)) .setImagetype(mergeString(to.clusterDefaults.imagetype, from.clusterDefaults.imagetype)) .setProvider(mergeString(to.clusterDefaults.provider, from.clusterDefaults.provider)) .build(); } //merge compatibilities services if(!from.compatibilities.equals(Compatibilities.EMPTY_COMPATIBILITIES)) { to.compatibilities = Compatibilities.builder() .setServices(mergeSet(to.compatibilities.services, from.compatibilities.services)) .setHardwaretypes(mergeSet(to.compatibilities.hardwaretypes, from.compatibilities.hardwaretypes)) .setImagetypes(mergeSet(to.compatibilities.imagetypes, from.compatibilities.imagetypes)) .build(); } }
@Test public void test_03_persistentTemplatesConsistence() throws Exception { PartialTemplate ldapInternal = entityStoreView.getPartialTemplate("LDAP-internal"); PartialTemplate sensuInternal = entityStoreView.getPartialTemplate("sensu-internal"); ClusterTemplate cdapDistributedSecureHadoop = entityStoreView.getClusterTemplate("cdap-distributed-secure-hadoop"); ClusterTemplate cdapDistributedInsecure = entityStoreView.getClusterTemplate("cdap-distributed-insecure"); ClusterTemplate cdapDistributed = entityStoreView.getClusterTemplate("cdap-distributed"); Assert.assertNotNull(ldapInternal); Assert.assertNotNull(sensuInternal); Assert.assertNotNull(cdapDistributedSecureHadoop); Assert.assertNotNull(cdapDistributedInsecure); Assert.assertNotNull(cdapDistributed); Assert.assertEquals("Configure Example, Inc. LDAP services", ldapInternal.getDescription()); Assert.assertEquals(true, sensuInternal.isImmutable()); Assert.assertEquals("ldap-internal", ldapInternal.clusterDefaults.getServices().iterator().next()); Assert.assertEquals("ldap-internal", ldapInternal.compatibilities.getServices().iterator().next()); Assert.assertNotNull(ldapInternal.clusterDefaults.getConfig().get("ldap")); Assert.assertEquals("ldap.wrong.com", ldapInternal.clusterDefaults.getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", cdapDistributedSecureHadoop.getDescription()); Assert.assertNotNull(cdapDistributedSecureHadoop.getParent()); Assert.assertEquals("cdap-distributed", cdapDistributedSecureHadoop.getParent().getName()); Assert.assertNotNull(cdapDistributedSecureHadoop.getIncludes()); Assert.assertEquals("LDAP-internal", cdapDistributedSecureHadoop.getIncludes().iterator().next().getName()); Assert.assertEquals(3, cdapDistributedSecureHadoop.clusterDefaults.getServices().size()); Assert.assertNotNull(cdapDistributedSecureHadoop.getClusterDefaults().getConfig().get("hive")); Assert.assertEquals("kerberos-client", cdapDistributedSecureHadoop.getCompatibilities().getServices().iterator().next()); }
public Compatibilities build() { return new Compatibilities(hardwaretypes, imagetypes, services); } }
private void addProviderFlavor(Map<String, String> map, String providerName, Compatibilities compatibilities, HardwareType hardwareType) { if (hardwareType != null) { Map<String, Map<String, String>> providerMap = hardwareType.getProviderMap(); String name = hardwareType.getName(); // empty allowed types means all types are allowed if (compatibilities.compatibleWithHardwareType(name) && providerMap.containsKey(providerName)) { String flavor = providerMap.get(providerName).get("flavor"); if (flavor != null) { map.put(name, flavor); } } } }
private void addProviderImage(Map<String, Map<String, String>> map, String providerName, Compatibilities compatibilities, ImageType imageType) { if (imageType != null) { Map<String, Map<String, String>> providerMap = imageType.getProviderMap(); String name = imageType.getName(); // empty allowed types means all types are allowed if (compatibilities.compatibleWithImageType(name) && providerMap.containsKey(providerName)) { Map<String, String> providerProperties = providerMap.get(providerName); String image = providerProperties.get("image"); if (image != null) { map.put(name, providerProperties); } } } }
Set<String> newCompatibleServices = Sets.newHashSet(template.getCompatibilities().getServices()); newCompatibleServices.add("new-service"); Compatibilities newCompatibilities = Compatibilities.builder() .setHardwaretypes(template.getCompatibilities().getHardwaretypes()) .setImagetypes(template.getCompatibilities().getImagetypes()) .setServices(newCompatibleServices).build(); ClusterTemplate updatedTemplate = ClusterTemplate.builder()
@Test public void test_07_not_persisted_resolveTemplate() throws Exception { InputStream notPersistedIn = classLoader.getResourceAsStream("partials/cdap-not-persisted.json"); ClusterTemplate notPersisted = gson.fromJson(IOUtils.toString(notPersistedIn), ClusterTemplate.class); ClusterTemplate rt = clusterService.resolveTemplate(account, notPersisted); Assert.assertEquals(notPersisted.getIncludes(), rt.getIncludes()); Assert.assertEquals(notPersisted.getParent(), rt.getParent()); Assert.assertEquals(notPersisted.getName(), rt.getName()); Assert.assertEquals(notPersisted.getDescription(), rt.getDescription()); Assert.assertEquals(sensuPartial.getClusterDefaults().getServices(), rt.getClusterDefaults().getServices()); Assert.assertEquals(sensuPartial.getClusterDefaults().getConfig(), rt.getClusterDefaults().getConfig()); Assert.assertEquals(sensuPartial.getCompatibilities().getServices(), rt.getCompatibilities().getServices()); }
@Test public void testNodesMustHaveServices() throws Exception { Set<String> services = ImmutableSet.of("svc1", "svc2", "svc3"); ClusterTemplate template = ClusterTemplate.builder() .setName("simple") .setDescription("all services on all nodes template") .setClusterDefaults(ClusterDefaults.builder().setServices(services).setProvider("joyent").build()) .setCompatibilities(Compatibilities.builder().setServices(services).build()) .setConstraints(new Constraints( ImmutableMap.<String, ServiceConstraint>of("svc1", new ServiceConstraint(null, null, 1, 1)), new LayoutConstraint( ImmutableSet.<Set<String>>of(ImmutableSet.of("svc1", "svc2", "svc3")), ImmutableSet.<Set<String>>of() ), SizeConstraint.EMPTY )).build(); List<NodeLayout> expected = ImmutableList.of( new NodeLayout("small", "centos6", ImmutableSet.of("svc1", "svc2", "svc3")) ); NodeLayoutGenerator nodeLayoutGenerator = new NodeLayoutGenerator(template, services, ImmutableSet.<String>of("small"), ImmutableSet.<String>of("centos6")); List<NodeLayout> actual = nodeLayoutGenerator.generateNodeLayoutPreferences(); Assert.assertEquals(expected, actual); }
@Test public void testNoSolutionReturnsNull() { Set<String> services = ImmutableSet.of("svc1", "svc2", "svc3"); ClusterTemplate template = ClusterTemplate.builder() .setName("simple") .setDescription("all services on all nodes template") .setClusterDefaults(ClusterDefaults.builder().setServices(services).setProvider("joyent").build()) .setCompatibilities(Compatibilities.builder().setServices(services).build()) .setConstraints(new Constraints( ImmutableMap.<String, ServiceConstraint>of("svc1", new ServiceConstraint(null, null, 1, 1)), new LayoutConstraint( ImmutableSet.<Set<String>>of(ImmutableSet.of("svc1", "svc2", "svc3")), ImmutableSet.<Set<String>>of() ), SizeConstraint.EMPTY)) .build(); List<NodeLayout> nodePreferences = ImmutableList.of( new NodeLayout("small", "centos6", ImmutableSet.of("svc1", "svc2", "svc3")) ); ClusterLayoutFinder finder = new ClusterLayoutFinder(nodePreferences, template, services, 2); Assert.assertNull(finder.findValidNodeCounts()); }
@BeforeClass public static void initData() throws Exception { JsonObject defaultClusterConfig = new JsonObject(); defaultClusterConfig.addProperty("defaultconfig", "value1"); smallTemplate = ClusterTemplate.builder() .setName("one-machine") .setClusterDefaults( ClusterDefaults.builder() .setServices("zookeeper") .setProvider("rackspace") .setConfig(defaultClusterConfig) .build()) .setCompatibilities(Compatibilities.builder().setServices("zookeeper").build()) .setAdministration(new Administration(LeaseDuration.of("10s", "30s", "5s"))) .build(); }
.setClusterDefaults(ClusterDefaults.builder().setServices(services).setProvider("joyent").build()) .setCompatibilities( Compatibilities.builder().setHardwaretypes("large-mem", "large-cpu", "large", "medium", "small").build()) .setConstraints(new Constraints( ImmutableMap.<String, ServiceConstraint>of(
.setClusterDefaults(ClusterDefaults.builder().setServices(services).setProvider("joyent").build()) .setCompatibilities( Compatibilities.builder().setHardwaretypes("large-mem", "large-cpu", "large", "medium", "small").build()) .setConstraints( new Constraints( .setConfig(defaultClusterConfig) .build()) .setCompatibilities(Compatibilities.builder().setServices("zookeeper").build()) .setAdministration(new Administration(LeaseDuration.of("10s", "30s", "5s"))) .build();
.setProvider("joyent") .setConfig(Entities.ClusterTemplateExample.clusterConf).build()) .setCompatibilities(Compatibilities.builder().setServices(services).build()) .setConstraints(new Constraints( ImmutableMap.<String, ServiceConstraint>of(