public ClusterLayoutFinder(List<NodeLayout> nodePreferences, ClusterTemplate template, Set<String> services, int numMachines) { this.nodePreferences = nodePreferences; this.numMachines = numMachines; serviceConstraints = Maps.newHashMap(); // we only care about the constraints that apply to services that are on the cluster Map<String, ServiceConstraint> allServiceConstraints = template.getConstraints().getServiceConstraints(); for (String service : services) { if (allServiceConstraints.containsKey(service)) { serviceConstraints.put(service, allServiceConstraints.get(service)); } } // calculate number of each service across the entire cluster serviceCounts = Maps.newHashMap(); for (String service : services) { serviceCounts.put(service, 0); } }
public ClusterLayoutTracker addServicesToCluster(Cluster cluster, Set<Node> clusterNodes, Set<String> servicesToAdd) throws Exception { Preconditions.checkArgument(cluster != null, "Cannot add services to a nonexistant cluster."); Preconditions.checkArgument(clusterNodes != null && !clusterNodes.isEmpty(), "Cannot add services to nonexistant nodes."); Constraints clusterConstraints = cluster.getClusterTemplate().getConstraints(); ClusterLayout clusterLayout = ClusterLayout.fromNodes(clusterNodes, clusterConstraints); Set<String> servicesToAddCopy = Sets.newHashSet(servicesToAdd); SortedSet<Map.Entry<String, ServiceConstraint>> sortedConstraints = Sets.newTreeSet(serviceComparator); sortedConstraints.addAll(clusterConstraints.getServiceConstraints().entrySet()); Queue<String> sortedServices = Lists.newLinkedList(); for (Map.Entry<String, ServiceConstraint> entry : sortedConstraints) { if (servicesToAddCopy.contains(entry.getKey())) { sortedServices.add(entry.getKey()); servicesToAddCopy.remove(entry.getKey()); } } // any service without a constraint has no limit on the number of nodes it can be placed on, so add them to the end sortedServices.addAll(servicesToAddCopy); ClusterLayoutTracker tracker = new ClusterLayoutTracker(clusterLayout); return canAddServicesToCluster(tracker, sortedServices) ? tracker : null; }
@Override public JsonElement serialize(Constraints constraints, Type type, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); jsonObj.add("layout", context.serialize(constraints.getLayoutConstraint())); jsonObj.add("services", context.serialize(constraints.getServiceConstraints())); jsonObj.add("size", context.serialize(constraints.getSizeConstraint())); return jsonObj; }
Set<NodeLayout> findValidNodeLayouts(Set<Set<String>> validServiceSets) { Set<NodeLayout> validNodeLayouts = Sets.newHashSet(); Map<String, ServiceConstraint> serviceConstraints = clusterTemplate.getConstraints().getServiceConstraints(); for (String hardwareType : allowedHardwareTypes) { for (String imageType : allowedImageTypes) { for (Set<String> serviceSet : validServiceSets) { NodeLayout nodeLayout = new NodeLayout(hardwareType, imageType, serviceSet); if (nodeLayout.satisfiesServiceConstraints(serviceConstraints)) { validNodeLayouts.add(nodeLayout); } } } } return validNodeLayouts; }
/** * Determine if this is a valid node layout some constraints and the set of all services that are also on the cluster. * * @param constraints Constraints to use for checking validity. * @param clusterServices Services on the cluster with this node layout. * @return True if it satisfies all constraints, false if not. */ public boolean satisfiesConstraints(Constraints constraints, Set<String> clusterServices) { return NodeLayoutGenerator.isValidServiceSet(services, constraints.getLayoutConstraint(), clusterServices) && satisfiesServiceConstraints(constraints.getServiceConstraints()); }
Set<String> findUnconstrainedServices() { Set<String> unconstrained = Sets.newHashSet(); Map<String, ServiceConstraint> serviceConstraints = clusterTemplate.getConstraints().getServiceConstraints(); LayoutConstraint layoutConstraint = clusterTemplate.getConstraints().getLayoutConstraint(); Set<Set<String>> mustCoexistServices = layoutConstraint.getServicesThatMustCoexist(); Set<Set<String>> mustNotCoexistServices = layoutConstraint.getServicesThatMustNotCoexist(); for (String service : clusterServices) { if (!serviceConstraints.containsKey(service) && !inSetOfSets(service, mustCoexistServices) && !inSetOfSets(service, mustNotCoexistServices)) { unconstrained.add(service); } } return unconstrained; }
ServiceConstraint serviceConstraint = clusterLayout.getConstraints().getServiceConstraints().get(service);
private boolean satisfiesConstraints(Constraints constraints) { // check node layouts Set<String> clusterServices = serviceCounts.elementSet(); for (NodeLayout nodeLayout : layout.elementSet()) { if (!nodeLayout.satisfiesConstraints(constraints, clusterServices)) { return false; } } // check service counts Map<String, ServiceConstraint> serviceConstraints = constraints.getServiceConstraints(); for (Multiset.Entry<String> entry : serviceCounts.entrySet()) { ServiceConstraint constraint = serviceConstraints.get(entry.getElement()); if (constraint != null) { int serviceCount = entry.getCount(); // TODO: ratio constraint if (serviceCount < constraint.getMinCount() || serviceCount > constraint.getMaxCount() || serviceCount > layout.size()) { return false; } } } return true; }
@Test public void testIsValidNodeLayout() { Map<String, ServiceConstraint> serviceConstraints = reactorTemplate.getConstraints().getServiceConstraints(); // test all possible valid node layouts Set<String> masterServices = ImmutableSet.of("namenode", "resourcemanager", "hbasemaster"); assertSatisfiesServiceConstraints("large-mem", "centos6", masterServices, serviceConstraints); assertSatisfiesServiceConstraints("large-mem", "ubuntu12", masterServices, serviceConstraints); Set<String> slaveServices = ImmutableSet.of("datanode", "nodemanager", "regionserver"); assertSatisfiesServiceConstraints("medium", "centos6", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("medium", "ubuntu12", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("large-cpu", "centos6", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("large-cpu", "ubuntu12", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("small", "centos6", ImmutableSet.of("zookeeper"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("zookeeper"), serviceConstraints); assertSatisfiesServiceConstraints("large", "centos6", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("large", "ubuntu12", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "ubuntu12", ImmutableSet.of("reactor"), serviceConstraints); // there are no image type constraints for reactor so this should pass assertSatisfiesServiceConstraints("medium", "asdf", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("reactor", "zookeeper"), serviceConstraints); // test hardware type is invalid assertUnsatisfiesServiceConstraints("large", "centos6", masterServices, serviceConstraints); // test image type is invalid assertUnsatisfiesServiceConstraints("large", "asdf", masterServices, serviceConstraints); // test both are invalid assertUnsatisfiesServiceConstraints("large-mem", "ubuntu12", ImmutableSet.of("reactor", "zookeeper"), serviceConstraints); }
@Test public void test_06_resolveTemplateTest() throws Exception { ClusterTemplate rt = clusterService.resolveTemplate(account, secureTemplate); Assert.assertEquals(secureTemplate.getIncludes(), rt.getIncludes()); Assert.assertEquals(secureTemplate.getParent(), rt.getParent()); Assert.assertEquals(secureTemplate.getName(), rt.getName()); Assert.assertEquals(secureTemplate.getDescription(), rt.getDescription()); Assert.assertEquals(defaultsServices, rt.getClusterDefaults().getServices()); Assert.assertEquals(16, rt.getClusterDefaults().getConfig().entrySet().size()); Assert.assertEquals("rackspace", rt.getClusterDefaults().getProvider()); Assert.assertEquals("standard-large", rt.getClusterDefaults().getHardwaretype()); Assert.assertEquals("centos6", rt.getClusterDefaults().getImagetype()); Assert.assertEquals("example.com", rt.getClusterDefaults().getDnsSuffix()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustCoexist().size()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustNotCoexist().size()); Assert.assertEquals(serviceConstraints, rt.getConstraints().getServiceConstraints()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getInitial()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getStep()); Assert.assertEquals(compatibilitiesHardwaretypes, rt.getCompatibilities().getHardwaretypes()); Assert.assertEquals(compatibilitiesImagetypes, rt.getCompatibilities().getImagetypes()); Assert.assertEquals(compatibilitiesServices, rt.getCompatibilities().getServices()); //overrides in child Assert.assertEquals("ldap.correct.com", rt.getClusterDefaults().getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertNull(rt.getLabel()); Assert.assertNull(rt.getIcon()); Assert.assertEquals("cdap-distributed-secure-hadoop", rt.getName()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", rt.getDescription()); }