public ClusterLayoutFinder(List<NodeLayout> nodePreferences, ClusterTemplate template, Set<String> services, int numMachines) { this.nodePreferences = nodePreferences; this.numMachines = numMachines; serviceConstraints = Maps.newHashMap(); // we only care about the constraints that apply to services that are on the cluster Map<String, ServiceConstraint> allServiceConstraints = template.getConstraints().getServiceConstraints(); for (String service : services) { if (allServiceConstraints.containsKey(service)) { serviceConstraints.put(service, allServiceConstraints.get(service)); } } // calculate number of each service across the entire cluster serviceCounts = Maps.newHashMap(); for (String service : services) { serviceCounts.put(service, 0); } }
public ClusterLayoutTracker addServicesToCluster(Cluster cluster, Set<Node> clusterNodes, Set<String> servicesToAdd) throws Exception { Preconditions.checkArgument(cluster != null, "Cannot add services to a nonexistant cluster."); Preconditions.checkArgument(clusterNodes != null && !clusterNodes.isEmpty(), "Cannot add services to nonexistant nodes."); Constraints clusterConstraints = cluster.getClusterTemplate().getConstraints(); ClusterLayout clusterLayout = ClusterLayout.fromNodes(clusterNodes, clusterConstraints); Set<String> servicesToAddCopy = Sets.newHashSet(servicesToAdd); SortedSet<Map.Entry<String, ServiceConstraint>> sortedConstraints = Sets.newTreeSet(serviceComparator); sortedConstraints.addAll(clusterConstraints.getServiceConstraints().entrySet()); Queue<String> sortedServices = Lists.newLinkedList(); for (Map.Entry<String, ServiceConstraint> entry : sortedConstraints) { if (servicesToAddCopy.contains(entry.getKey())) { sortedServices.add(entry.getKey()); servicesToAddCopy.remove(entry.getKey()); } } // any service without a constraint has no limit on the number of nodes it can be placed on, so add them to the end sortedServices.addAll(servicesToAddCopy); ClusterLayoutTracker tracker = new ClusterLayoutTracker(clusterLayout); return canAddServicesToCluster(tracker, sortedServices) ? tracker : null; }
Set<String> findUnconstrainedServices() { Set<String> unconstrained = Sets.newHashSet(); Map<String, ServiceConstraint> serviceConstraints = clusterTemplate.getConstraints().getServiceConstraints(); LayoutConstraint layoutConstraint = clusterTemplate.getConstraints().getLayoutConstraint(); Set<Set<String>> mustCoexistServices = layoutConstraint.getServicesThatMustCoexist(); Set<Set<String>> mustNotCoexistServices = layoutConstraint.getServicesThatMustNotCoexist(); for (String service : clusterServices) { if (!serviceConstraints.containsKey(service) && !inSetOfSets(service, mustCoexistServices) && !inSetOfSets(service, mustNotCoexistServices)) { unconstrained.add(service); } } return unconstrained; }
Set<NodeLayout> findValidNodeLayouts(Set<Set<String>> validServiceSets) { Set<NodeLayout> validNodeLayouts = Sets.newHashSet(); Map<String, ServiceConstraint> serviceConstraints = clusterTemplate.getConstraints().getServiceConstraints(); for (String hardwareType : allowedHardwareTypes) { for (String imageType : allowedImageTypes) { for (Set<String> serviceSet : validServiceSets) { NodeLayout nodeLayout = new NodeLayout(hardwareType, imageType, serviceSet); if (nodeLayout.satisfiesServiceConstraints(serviceConstraints)) { validNodeLayouts.add(nodeLayout); } } } } return validNodeLayouts; }
LayoutConstraint layoutConstraint = clusterTemplate.getConstraints().getLayoutConstraint(); int[] maxCounts = new int[services.size()]; for (int i = 0; i < maxCounts.length; i++) {
public boolean isCompatibleWithTemplate(ClusterTemplate template) { // check all services are compatible Compatibilities compatibilities = template.getCompatibilities(); if (!compatibilities.compatibleWithServices(serviceCounts.elementSet())) { return false; } // check all image types and hardware types are compatible Set<String> imageTypes = Sets.newHashSet(); Set<String> hardwareTypes = Sets.newHashSet(); for (NodeLayout nodeLayout : layout.elementSet()) { imageTypes.add(nodeLayout.getImageTypeName()); hardwareTypes.add(nodeLayout.getHardwareTypeName()); } // check all hardware types are compatible if (!compatibilities.compatibleWithImageTypes(imageTypes) || !compatibilities.compatibleWithHardwareTypes(hardwareTypes)) { return false; } return satisfiesConstraints(template.getConstraints()); }
@Test public void testAddService() throws Exception { // adding resourcemanager Multiset<NodeLayout> expectedCounts = HashMultiset.create(); NodeLayout masterNodeLayout = new NodeLayout("large-mem", "centos6", ImmutableSet.of(namenode.getName(), resourcemanager.getName())); expectedCounts.add(masterNodeLayout, 1); NodeLayout slaveNodeLayout = new NodeLayout("medium", "centos6", ImmutableSet.of(datanode.getName())); expectedCounts.add(slaveNodeLayout, 50); ClusterLayout expected = new ClusterLayout(reactorTemplate.getConstraints(), expectedCounts); ClusterLayout layout = updater.addServicesToCluster(cluster, nodes, ImmutableSet.of(resourcemanager.getName())).getCurrentLayout(); Assert.assertEquals(expected, layout); }
throw new MissingEntityException("could not find cluster nodes"); ClusterLayout clusterLayout = ClusterLayout.fromNodes(clusterNodes, currentTemplate.getConstraints());
@Test public void testIsValidNodeLayout() { Map<String, ServiceConstraint> serviceConstraints = reactorTemplate.getConstraints().getServiceConstraints(); // test all possible valid node layouts Set<String> masterServices = ImmutableSet.of("namenode", "resourcemanager", "hbasemaster"); assertSatisfiesServiceConstraints("large-mem", "centos6", masterServices, serviceConstraints); assertSatisfiesServiceConstraints("large-mem", "ubuntu12", masterServices, serviceConstraints); Set<String> slaveServices = ImmutableSet.of("datanode", "nodemanager", "regionserver"); assertSatisfiesServiceConstraints("medium", "centos6", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("medium", "ubuntu12", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("large-cpu", "centos6", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("large-cpu", "ubuntu12", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("small", "centos6", ImmutableSet.of("zookeeper"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("zookeeper"), serviceConstraints); assertSatisfiesServiceConstraints("large", "centos6", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("large", "ubuntu12", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "ubuntu12", ImmutableSet.of("reactor"), serviceConstraints); // there are no image type constraints for reactor so this should pass assertSatisfiesServiceConstraints("medium", "asdf", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("reactor", "zookeeper"), serviceConstraints); // test hardware type is invalid assertUnsatisfiesServiceConstraints("large", "centos6", masterServices, serviceConstraints); // test image type is invalid assertUnsatisfiesServiceConstraints("large", "asdf", masterServices, serviceConstraints); // test both are invalid assertUnsatisfiesServiceConstraints("large-mem", "ubuntu12", ImmutableSet.of("reactor", "zookeeper"), serviceConstraints); }
@Test public void testAddServices() throws Exception { // adding resourcemanager, nodemanager, hbasemaster, regionserver Multiset<NodeLayout> expectedCounts = HashMultiset.create(); NodeLayout masterNodeLayout = new NodeLayout("large-mem", "centos6", ImmutableSet.of(namenode.getName(), resourcemanager.getName(), hbasemaster.getName())); expectedCounts.add(masterNodeLayout, 1); NodeLayout slaveNodeLayout = new NodeLayout("medium", "centos6", ImmutableSet.of(datanode.getName(), nodemanager.getName(), regionserver.getName())); expectedCounts.add(slaveNodeLayout, 50); ClusterLayout expected = new ClusterLayout(reactorTemplate.getConstraints(), expectedCounts); ClusterLayout layout = updater.addServicesToCluster( cluster, nodes, ImmutableSet.of(resourcemanager.getName(), nodemanager.getName(), hbasemaster.getName(), regionserver.getName())).getCurrentLayout(); Assert.assertEquals(expected, layout); }
@Test public void testIsValidServiceSet() { LayoutConstraint layoutConstraint = reactorTemplate.getConstraints().getLayoutConstraint(); Set<String> services = ImmutableSet.of("namenode", "datanode", "regionserver", "nodemanager", "hbasemaster", "reactor", "zookeeper"); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("datanode", "regionserver", "nodemanager"), layoutConstraint, services)); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("namenode", "resourcemanager", "hbasemaster"), layoutConstraint, services)); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("reactor", "zookeeper"), layoutConstraint, services)); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("reactor"), layoutConstraint, services)); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("zookeeper"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("datanode"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("datanode", "nodemanager"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("namenode"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("namenode", "resourcemanager"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("namenode", "datanode"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("reactor", "datanode"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("zookeeper", "datanode"), layoutConstraint, services)); }
SizeConstraint sizeConstraint = template.getConstraints().getSizeConstraint(); sizeConstraint.verify(numMachines); builder.setClusterTemplate(template);
.setServices(ImmutableSet.of(namenode.getName(), datanode.getName())) .build(); Constraints constraints = cluster.getClusterTemplate().getConstraints();
private ClusterTemplate copyWithNewCompatibilities(ClusterTemplate template, Compatibilities newCompatibilities) { return ClusterTemplate.builder() .setName(template.getName()) .setDescription(template.getDescription()) .setClusterDefaults(template.getClusterDefaults()) .setCompatibilities(newCompatibilities) .setConstraints(template.getConstraints()) .setAdministration(template.getAdministration()) .build(); }
@Test public void test_06_resolveTemplateTest() throws Exception { ClusterTemplate rt = clusterService.resolveTemplate(account, secureTemplate); Assert.assertEquals(secureTemplate.getIncludes(), rt.getIncludes()); Assert.assertEquals(secureTemplate.getParent(), rt.getParent()); Assert.assertEquals(secureTemplate.getName(), rt.getName()); Assert.assertEquals(secureTemplate.getDescription(), rt.getDescription()); Assert.assertEquals(defaultsServices, rt.getClusterDefaults().getServices()); Assert.assertEquals(16, rt.getClusterDefaults().getConfig().entrySet().size()); Assert.assertEquals("rackspace", rt.getClusterDefaults().getProvider()); Assert.assertEquals("standard-large", rt.getClusterDefaults().getHardwaretype()); Assert.assertEquals("centos6", rt.getClusterDefaults().getImagetype()); Assert.assertEquals("example.com", rt.getClusterDefaults().getDnsSuffix()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustCoexist().size()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustNotCoexist().size()); Assert.assertEquals(serviceConstraints, rt.getConstraints().getServiceConstraints()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getInitial()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getStep()); Assert.assertEquals(compatibilitiesHardwaretypes, rt.getCompatibilities().getHardwaretypes()); Assert.assertEquals(compatibilitiesImagetypes, rt.getCompatibilities().getImagetypes()); Assert.assertEquals(compatibilitiesServices, rt.getCompatibilities().getServices()); //overrides in child Assert.assertEquals("ldap.correct.com", rt.getClusterDefaults().getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertNull(rt.getLabel()); Assert.assertNull(rt.getIcon()); Assert.assertEquals("cdap-distributed-secure-hadoop", rt.getName()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", rt.getDescription()); }
counts.add(reactorLayout); counts.add(slaveLayout, 50); ClusterLayout layout = new ClusterLayout(template.getConstraints(), counts);
.setClusterDefaults(template.getClusterDefaults()) .setCompatibilities(newCompatibilities) .setConstraints(template.getConstraints()) .setAdministration(template.getAdministration()) .build();
.setClusterDefaults(template.getClusterDefaults()) .setCompatibilities(newCompatibilities) .setConstraints(template.getConstraints()) .setAdministration(template.getAdministration()) .build();