private ClusterTemplate copyWithNewCompatibilities(ClusterTemplate template, Compatibilities newCompatibilities) { return ClusterTemplate.builder() .setName(template.getName()) .setDescription(template.getDescription()) .setClusterDefaults(template.getClusterDefaults()) .setCompatibilities(newCompatibilities) .setConstraints(template.getConstraints()) .setAdministration(template.getAdministration()) .build(); }
@Test public void test_06_resolveTemplateTest() throws Exception { ClusterTemplate rt = clusterService.resolveTemplate(account, secureTemplate); Assert.assertEquals(secureTemplate.getIncludes(), rt.getIncludes()); Assert.assertEquals(secureTemplate.getParent(), rt.getParent()); Assert.assertEquals(secureTemplate.getName(), rt.getName()); Assert.assertEquals(secureTemplate.getDescription(), rt.getDescription()); Assert.assertEquals(defaultsServices, rt.getClusterDefaults().getServices()); Assert.assertEquals(16, rt.getClusterDefaults().getConfig().entrySet().size()); Assert.assertEquals("rackspace", rt.getClusterDefaults().getProvider()); Assert.assertEquals("standard-large", rt.getClusterDefaults().getHardwaretype()); Assert.assertEquals("centos6", rt.getClusterDefaults().getImagetype()); Assert.assertEquals("example.com", rt.getClusterDefaults().getDnsSuffix()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustCoexist().size()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustNotCoexist().size()); Assert.assertEquals(serviceConstraints, rt.getConstraints().getServiceConstraints()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getInitial()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getStep()); Assert.assertEquals(compatibilitiesHardwaretypes, rt.getCompatibilities().getHardwaretypes()); Assert.assertEquals(compatibilitiesImagetypes, rt.getCompatibilities().getImagetypes()); Assert.assertEquals(compatibilitiesServices, rt.getCompatibilities().getServices()); //overrides in child Assert.assertEquals("ldap.correct.com", rt.getClusterDefaults().getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertNull(rt.getLabel()); Assert.assertNull(rt.getIcon()); Assert.assertEquals("cdap-distributed-secure-hadoop", rt.getName()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", rt.getDescription()); }
@Override public void writeClusterTemplate(ClusterTemplate clusterTemplate) throws IOException, IllegalAccessException { int version = getVersion(EntityType.CLUSTER_TEMPLATE, clusterTemplate.getName()); clusterTemplate.setVersion(version); writeEntity(EntityType.CLUSTER_TEMPLATE, clusterTemplate.getName(), version, serialize(clusterTemplate, ClusterTemplate.class)); clusterTemplate.setVersion(Constants.DEFAULT_VERSION); }
@Override protected void addChildFields(ClusterTemplate template, JsonObject jsonObj, JsonSerializationContext context) { super.addChildFields(template, jsonObj, context); jsonObj.add(EXTENDS_KEY, context.serialize(template.getParent())); jsonObj.add(INCLUDES_KEY, context.serialize(template.getIncludes())); }
@Test public void test_07_not_persisted_resolveTemplate() throws Exception { InputStream notPersistedIn = classLoader.getResourceAsStream("partials/cdap-not-persisted.json"); ClusterTemplate notPersisted = gson.fromJson(IOUtils.toString(notPersistedIn), ClusterTemplate.class); ClusterTemplate rt = clusterService.resolveTemplate(account, notPersisted); Assert.assertEquals(notPersisted.getIncludes(), rt.getIncludes()); Assert.assertEquals(notPersisted.getParent(), rt.getParent()); Assert.assertEquals(notPersisted.getName(), rt.getName()); Assert.assertEquals(notPersisted.getDescription(), rt.getDescription()); Assert.assertEquals(sensuPartial.getClusterDefaults().getServices(), rt.getClusterDefaults().getServices()); Assert.assertEquals(sensuPartial.getClusterDefaults().getConfig(), rt.getClusterDefaults().getConfig()); Assert.assertEquals(sensuPartial.getCompatibilities().getServices(), rt.getCompatibilities().getServices()); }
@Test public void testClusterCreateLeaseDuration() throws Exception { // Reactor template has forever initial lease duration verifyInitialLeaseDuration(200, HttpResponseStatus.OK, 200, reactorTemplate.getName()); verifyInitialLeaseDuration(0, HttpResponseStatus.OK, 0, reactorTemplate.getName()); verifyInitialLeaseDuration(0, HttpResponseStatus.OK, -1, reactorTemplate.getName()); // Small template has 10000 initial lease duration verifyInitialLeaseDuration(10000, HttpResponseStatus.BAD_REQUEST, 20000, smallTemplate.getName()); verifyInitialLeaseDuration(10000, HttpResponseStatus.OK, 10000, smallTemplate.getName()); verifyInitialLeaseDuration(500, HttpResponseStatus.OK, 500, smallTemplate.getName()); verifyInitialLeaseDuration(10000, HttpResponseStatus.BAD_REQUEST, 0, smallTemplate.getName()); verifyInitialLeaseDuration(10000, HttpResponseStatus.OK, -1, smallTemplate.getName()); }
throws TemplateImmutabilityException, TemplateNotFoundException, TemplateValidationException { Set<String> immutables = Sets.newHashSet(); ClusterTemplate toTemplate = ClusterTemplate.builder() .setName(initialTemplate.getName()) .setDescription(initialTemplate.getDescription()) .setIcon(initialTemplate.getIcon()) .setLabel(initialTemplate.getLabel()) .build(); toTemplate.includes = initialTemplate.includes;
@Override protected AbstractTemplate.Builder getConcreteBuilder() { return ClusterTemplate.builder(); } }
if (requiredHardwareType == null || requiredHardwareType.isEmpty()) { requiredHardwareType = template.getClusterDefaults().getHardwaretype(); if (hardwareTypeFlavors.isEmpty()) { throw new IllegalArgumentException("no hardware types are available to use with template " + template.getName() + " and provider " + providerName); if (requiredImageType == null || requiredImageType.isEmpty()) { requiredImageType = template.getClusterDefaults().getImagetype(); if (imageTypeMap.isEmpty()) { throw new IllegalArgumentException("no image types are available to use with template " + template.getName() + " and provider " + providerName); serviceNames = template.getClusterDefaults().getServices(); validateServiceCompatibilities(template.getCompatibilities(), serviceNames); dnsSuffix = template.getClusterDefaults().getDnsSuffix();
public ClusterLayoutFinder(List<NodeLayout> nodePreferences, ClusterTemplate template, Set<String> services, int numMachines) { this.nodePreferences = nodePreferences; this.numMachines = numMachines; serviceConstraints = Maps.newHashMap(); // we only care about the constraints that apply to services that are on the cluster Map<String, ServiceConstraint> allServiceConstraints = template.getConstraints().getServiceConstraints(); for (String service : services) { if (allServiceConstraints.containsKey(service)) { serviceConstraints.put(service, allServiceConstraints.get(service)); } } // calculate number of each service across the entire cluster serviceCounts = Maps.newHashMap(); for (String service : services) { serviceCounts.put(service, 0); } }
private Set<String> getServices(ClusterTemplate template, ClusterCreateRequest createRequest) { // set cluster service names. Dependency checking is done later on since it involves a lot of potential lookups. Set<String> serviceNames = createRequest.getServices(); if (serviceNames == null || serviceNames.isEmpty()) { serviceNames = template.getClusterDefaults().getServices(); } return serviceNames; }
public boolean isCompatibleWithTemplate(ClusterTemplate template) { // check all services are compatible Compatibilities compatibilities = template.getCompatibilities(); if (!compatibilities.compatibleWithServices(serviceCounts.elementSet())) { return false; } // check all image types and hardware types are compatible Set<String> imageTypes = Sets.newHashSet(); Set<String> hardwareTypes = Sets.newHashSet(); for (NodeLayout nodeLayout : layout.elementSet()) { imageTypes.add(nodeLayout.getImageTypeName()); hardwareTypes.add(nodeLayout.getHardwareTypeName()); } // check all hardware types are compatible if (!compatibilities.compatibleWithImageTypes(imageTypes) || !compatibilities.compatibleWithHardwareTypes(hardwareTypes)) { return false; } return satisfiesConstraints(template.getConstraints()); }
String templateName = cluster.getClusterTemplate().getName(); ClusterTemplate currentTemplate = entityStoreService.getView(account).getClusterTemplate(templateName); if (currentTemplate == null) { throw new MissingEntityException("could not find cluster nodes"); ClusterLayout clusterLayout = ClusterLayout.fromNodes(clusterNodes, currentTemplate.getConstraints());
@Test public void testInvalidNumMachines() throws Exception { // when its below the min ClusterCreateRequest clusterCreateRequest =ClusterCreateRequest.builder() .setName("my-cluster") .setClusterTemplateName(reactorTemplate.getName()) .setProviderName(reactorTemplate.getClusterDefaults().getProvider()) .setNumMachines(1) .build(); assertResponseStatus(doPostExternalAPI("/clusters", gson.toJson(clusterCreateRequest), USER1_HEADERS), HttpResponseStatus.BAD_REQUEST); // when its above the max clusterCreateRequest = ClusterCreateRequest.builder() .setName("my-cluster") .setClusterTemplateName(reactorTemplate.getName()) .setProviderName(reactorTemplate.getClusterDefaults().getProvider()) .setNumMachines(500) .build(); assertResponseStatus(doPostExternalAPI("/clusters", gson.toJson(clusterCreateRequest), USER1_HEADERS), HttpResponseStatus.BAD_REQUEST); }
SizeConstraint sizeConstraint = template.getConstraints().getSizeConstraint(); sizeConstraint.verify(numMachines); builder.setClusterTemplate(template); long leaseDuration = template.getAdministration().getLeaseDuration().calcInitialLease(requestedLease); long createTime = System.currentTimeMillis(); builder.setCreateTime(createTime);
private void validateServicesToAdd(Cluster cluster, Set<String> servicesToAdd, Map<String, Service> serviceMap) { validateServiceCompatibilities(cluster.getClusterTemplate().getCompatibilities(), servicesToAdd); validateServiceDependencies(serviceMap); }
@Test public void testGetStoreDeleteClusterTemplate() throws Exception { EntityStoreView entityStore = entityStoreService.getView(tenant1Admin); ClusterTemplate clusterTemplate = Entities.ClusterTemplateExample.REACTOR; String clusterTemplateName = clusterTemplate.getName(); Assert.assertNull(entityStore.getClusterTemplate(clusterTemplateName)); // write should work entityStore.writeClusterTemplate(clusterTemplate); ClusterTemplate result = entityStore.getClusterTemplate(clusterTemplateName); Assert.assertEquals(clusterTemplate, result); // bump version should work entityStore.writeClusterTemplate(clusterTemplate); entityStore.writeClusterTemplate(clusterTemplate); Collection<ClusterTemplate> clusterTemplates = entityStore.getAllClusterTemplates(); Assert.assertEquals(1, clusterTemplates.size()); Set<Integer> versions = Sets.newHashSet(); for (ClusterTemplate clusterTemplate1 : clusterTemplates) { versions.add(clusterTemplate1.getVersion()); } Assert.assertEquals(Sets.newHashSet(3), versions); // delete should work entityStore.deleteClusterTemplate(clusterTemplateName); Assert.assertNull(entityStore.getClusterTemplate(clusterTemplateName)); }
long step = cluster.getClusterTemplate().getAdministration().getLeaseDuration().getStep(); step = step == 0 ? Long.MAX_VALUE : step; long clusterMaxDuration = cluster.getClusterTemplate().getAdministration().getLeaseDuration().getMax(); if (clusterMaxDuration != 0 && expireTime - cluster.getExpireTime() > step) { throw new IllegalArgumentException("New expire time cannot be greater than allowed increment");
@Test public void test_03_persistentTemplatesConsistence() throws Exception { PartialTemplate ldapInternal = entityStoreView.getPartialTemplate("LDAP-internal"); PartialTemplate sensuInternal = entityStoreView.getPartialTemplate("sensu-internal"); ClusterTemplate cdapDistributedSecureHadoop = entityStoreView.getClusterTemplate("cdap-distributed-secure-hadoop"); ClusterTemplate cdapDistributedInsecure = entityStoreView.getClusterTemplate("cdap-distributed-insecure"); ClusterTemplate cdapDistributed = entityStoreView.getClusterTemplate("cdap-distributed"); Assert.assertNotNull(ldapInternal); Assert.assertNotNull(sensuInternal); Assert.assertNotNull(cdapDistributedSecureHadoop); Assert.assertNotNull(cdapDistributedInsecure); Assert.assertNotNull(cdapDistributed); Assert.assertEquals("Configure Example, Inc. LDAP services", ldapInternal.getDescription()); Assert.assertEquals(true, sensuInternal.isImmutable()); Assert.assertEquals("ldap-internal", ldapInternal.clusterDefaults.getServices().iterator().next()); Assert.assertEquals("ldap-internal", ldapInternal.compatibilities.getServices().iterator().next()); Assert.assertNotNull(ldapInternal.clusterDefaults.getConfig().get("ldap")); Assert.assertEquals("ldap.wrong.com", ldapInternal.clusterDefaults.getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", cdapDistributedSecureHadoop.getDescription()); Assert.assertNotNull(cdapDistributedSecureHadoop.getParent()); Assert.assertEquals("cdap-distributed", cdapDistributedSecureHadoop.getParent().getName()); Assert.assertNotNull(cdapDistributedSecureHadoop.getIncludes()); Assert.assertEquals("LDAP-internal", cdapDistributedSecureHadoop.getIncludes().iterator().next().getName()); Assert.assertEquals(3, cdapDistributedSecureHadoop.clusterDefaults.getServices().size()); Assert.assertNotNull(cdapDistributedSecureHadoop.getClusterDefaults().getConfig().get("hive")); Assert.assertEquals("kerberos-client", cdapDistributedSecureHadoop.getCompatibilities().getServices().iterator().next()); }