@Override public Constraints deserialize(JsonElement json, Type type, JsonDeserializationContext context) throws JsonParseException { JsonObject jsonObj = json.getAsJsonObject(); LayoutConstraint layoutConstraint = context.deserialize(jsonObj.get("layout"), LayoutConstraint.class); Map<String, ServiceConstraint> serviceConstraints = context.deserialize(jsonObj.get("services"), new TypeToken<Map<String, ServiceConstraint>>() { }.getType()); SizeConstraint sizeConstraint = context.deserialize(jsonObj.get("size"), SizeConstraint.class); return new Constraints(serviceConstraints, layoutConstraint, sizeConstraint); } }
@Override public JsonElement serialize(Constraints constraints, Type type, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); jsonObj.add("layout", context.serialize(constraints.getLayoutConstraint())); jsonObj.add("services", context.serialize(constraints.getServiceConstraints())); jsonObj.add("size", context.serialize(constraints.getSizeConstraint())); return jsonObj; }
/** * Determine if this is a valid node layout some constraints and the set of all services that are also on the cluster. * * @param constraints Constraints to use for checking validity. * @param clusterServices Services on the cluster with this node layout. * @return True if it satisfies all constraints, false if not. */ public boolean satisfiesConstraints(Constraints constraints, Set<String> clusterServices) { return NodeLayoutGenerator.isValidServiceSet(services, constraints.getLayoutConstraint(), clusterServices) && satisfiesServiceConstraints(constraints.getServiceConstraints()); }
public ClusterLayoutFinder(List<NodeLayout> nodePreferences, ClusterTemplate template, Set<String> services, int numMachines) { this.nodePreferences = nodePreferences; this.numMachines = numMachines; serviceConstraints = Maps.newHashMap(); // we only care about the constraints that apply to services that are on the cluster Map<String, ServiceConstraint> allServiceConstraints = template.getConstraints().getServiceConstraints(); for (String service : services) { if (allServiceConstraints.containsKey(service)) { serviceConstraints.put(service, allServiceConstraints.get(service)); } } // calculate number of each service across the entire cluster serviceCounts = Maps.newHashMap(); for (String service : services) { serviceCounts.put(service, 0); } }
private void copyFullProps(AbstractTemplate from, AbstractTemplate to, Set<String> immutables) throws TemplateImmutabilityException { copyMainProps(from, to, immutables, false); //merge constraints if(!from.constraints.equals(Constraints.EMPTY_CONSTRAINTS)) { Map<String, ServiceConstraint> serviceConstraint = mergeMap(to.constraints.serviceConstraints, from.constraints.serviceConstraints); Set<Set<String>> servicesThatMustCoexist = mergeSet(to.constraints.layoutConstraint.servicesThatMustCoexist, from.constraints.layoutConstraint.servicesThatMustCoexist); Set<Set<String>> servicesThatMustNotCoexist = mergeSet(to.constraints.layoutConstraint.servicesThatMustNotCoexist, from.constraints.layoutConstraint.servicesThatMustNotCoexist); to.constraints = new Constraints(serviceConstraint, new LayoutConstraint(servicesThatMustCoexist, servicesThatMustNotCoexist), from.constraints.sizeConstraint); } //merge admin lease duration if (!from.administration.equals(Administration.EMPTY_ADMINISTRATION)) { to.administration = from.administration; } }
LayoutConstraint layoutConstraint = clusterTemplate.getConstraints().getLayoutConstraint(); int[] maxCounts = new int[services.size()]; for (int i = 0; i < maxCounts.length; i++) {
SizeConstraint sizeConstraint = template.getConstraints().getSizeConstraint(); sizeConstraint.verify(numMachines); builder.setClusterTemplate(template);
Set<String> findUnconstrainedServices() { Set<String> unconstrained = Sets.newHashSet(); Map<String, ServiceConstraint> serviceConstraints = clusterTemplate.getConstraints().getServiceConstraints(); LayoutConstraint layoutConstraint = clusterTemplate.getConstraints().getLayoutConstraint(); Set<Set<String>> mustCoexistServices = layoutConstraint.getServicesThatMustCoexist(); Set<Set<String>> mustNotCoexistServices = layoutConstraint.getServicesThatMustNotCoexist(); for (String service : clusterServices) { if (!serviceConstraints.containsKey(service) && !inSetOfSets(service, mustCoexistServices) && !inSetOfSets(service, mustNotCoexistServices)) { unconstrained.add(service); } } return unconstrained; }
public ClusterLayoutTracker addServicesToCluster(Cluster cluster, Set<Node> clusterNodes, Set<String> servicesToAdd) throws Exception { Preconditions.checkArgument(cluster != null, "Cannot add services to a nonexistant cluster."); Preconditions.checkArgument(clusterNodes != null && !clusterNodes.isEmpty(), "Cannot add services to nonexistant nodes."); Constraints clusterConstraints = cluster.getClusterTemplate().getConstraints(); ClusterLayout clusterLayout = ClusterLayout.fromNodes(clusterNodes, clusterConstraints); Set<String> servicesToAddCopy = Sets.newHashSet(servicesToAdd); SortedSet<Map.Entry<String, ServiceConstraint>> sortedConstraints = Sets.newTreeSet(serviceComparator); sortedConstraints.addAll(clusterConstraints.getServiceConstraints().entrySet()); Queue<String> sortedServices = Lists.newLinkedList(); for (Map.Entry<String, ServiceConstraint> entry : sortedConstraints) { if (servicesToAddCopy.contains(entry.getKey())) { sortedServices.add(entry.getKey()); servicesToAddCopy.remove(entry.getKey()); } } // any service without a constraint has no limit on the number of nodes it can be placed on, so add them to the end sortedServices.addAll(servicesToAddCopy); ClusterLayoutTracker tracker = new ClusterLayoutTracker(clusterLayout); return canAddServicesToCluster(tracker, sortedServices) ? tracker : null; }
@Test public void testIsValidServiceSet() { LayoutConstraint layoutConstraint = reactorTemplate.getConstraints().getLayoutConstraint(); Set<String> services = ImmutableSet.of("namenode", "datanode", "regionserver", "nodemanager", "hbasemaster", "reactor", "zookeeper"); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("datanode", "regionserver", "nodemanager"), layoutConstraint, services)); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("namenode", "resourcemanager", "hbasemaster"), layoutConstraint, services)); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("reactor", "zookeeper"), layoutConstraint, services)); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("reactor"), layoutConstraint, services)); Assert.assertTrue(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("zookeeper"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("datanode"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("datanode", "nodemanager"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("namenode"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("namenode", "resourcemanager"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("namenode", "datanode"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("reactor", "datanode"), layoutConstraint, services)); Assert.assertFalse(NodeLayoutGenerator.isValidServiceSet( ImmutableSet.of("zookeeper", "datanode"), layoutConstraint, services)); }
@Test public void test_06_resolveTemplateTest() throws Exception { ClusterTemplate rt = clusterService.resolveTemplate(account, secureTemplate); Assert.assertEquals(secureTemplate.getIncludes(), rt.getIncludes()); Assert.assertEquals(secureTemplate.getParent(), rt.getParent()); Assert.assertEquals(secureTemplate.getName(), rt.getName()); Assert.assertEquals(secureTemplate.getDescription(), rt.getDescription()); Assert.assertEquals(defaultsServices, rt.getClusterDefaults().getServices()); Assert.assertEquals(16, rt.getClusterDefaults().getConfig().entrySet().size()); Assert.assertEquals("rackspace", rt.getClusterDefaults().getProvider()); Assert.assertEquals("standard-large", rt.getClusterDefaults().getHardwaretype()); Assert.assertEquals("centos6", rt.getClusterDefaults().getImagetype()); Assert.assertEquals("example.com", rt.getClusterDefaults().getDnsSuffix()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustCoexist().size()); Assert.assertEquals(2, rt.getConstraints().getLayoutConstraint().getServicesThatMustNotCoexist().size()); Assert.assertEquals(serviceConstraints, rt.getConstraints().getServiceConstraints()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getInitial()); Assert.assertEquals(43200000, rt.getAdministration().getLeaseDuration().getStep()); Assert.assertEquals(compatibilitiesHardwaretypes, rt.getCompatibilities().getHardwaretypes()); Assert.assertEquals(compatibilitiesImagetypes, rt.getCompatibilities().getImagetypes()); Assert.assertEquals(compatibilitiesServices, rt.getCompatibilities().getServices()); //overrides in child Assert.assertEquals("ldap.correct.com", rt.getClusterDefaults().getConfig().get("ldap") .getAsJsonObject().get("endpoint").getAsString()); Assert.assertNull(rt.getLabel()); Assert.assertNull(rt.getIcon()); Assert.assertEquals("cdap-distributed-secure-hadoop", rt.getName()); Assert.assertEquals("Cask DAP (CDAP) with Security and Secure Hadoop cluster with single master", rt.getDescription()); }
@BeforeClass public static void beforeClusterLayoutTest() { constraints = new Constraints( ImmutableMap.<String, ServiceConstraint>of( "namenode",
Set<NodeLayout> findValidNodeLayouts(Set<Set<String>> validServiceSets) { Set<NodeLayout> validNodeLayouts = Sets.newHashSet(); Map<String, ServiceConstraint> serviceConstraints = clusterTemplate.getConstraints().getServiceConstraints(); for (String hardwareType : allowedHardwareTypes) { for (String imageType : allowedImageTypes) { for (Set<String> serviceSet : validServiceSets) { NodeLayout nodeLayout = new NodeLayout(hardwareType, imageType, serviceSet); if (nodeLayout.satisfiesServiceConstraints(serviceConstraints)) { validNodeLayouts.add(nodeLayout); } } } } return validNodeLayouts; }
@BeforeClass public static void beforeClass() { constraints = new Constraints( ImmutableMap.<String, ServiceConstraint>of( "master1", new ServiceConstraint( ImmutableSet.of("large"), ImmutableSet.of("centos6", "ubuntu12"), 1, 1), "slave1", new ServiceConstraint( ImmutableSet.of("medium"), ImmutableSet.of("centos6", "ubuntu12"), 1, 50), "slave2", new ServiceConstraint( ImmutableSet.of("medium"), ImmutableSet.of("centos6", "ubuntu12"), 1, 50) ), new LayoutConstraint( ImmutableSet.<Set<String>>of( ImmutableSet.of("slave1", "slave2") ), ImmutableSet.<Set<String>>of( ImmutableSet.of("master1", "slave1"), ImmutableSet.of("master1", "slave2") ) ), SizeConstraint.EMPTY ); } }
ServiceConstraint serviceConstraint = clusterLayout.getConstraints().getServiceConstraints().get(service);
@BeforeClass public static void beforeClass() { constraints = new Constraints( ImmutableMap.<String, ServiceConstraint>of( "namenode",
private boolean satisfiesConstraints(Constraints constraints) { // check node layouts Set<String> clusterServices = serviceCounts.elementSet(); for (NodeLayout nodeLayout : layout.elementSet()) { if (!nodeLayout.satisfiesConstraints(constraints, clusterServices)) { return false; } } // check service counts Map<String, ServiceConstraint> serviceConstraints = constraints.getServiceConstraints(); for (Multiset.Entry<String> entry : serviceCounts.entrySet()) { ServiceConstraint constraint = serviceConstraints.get(entry.getElement()); if (constraint != null) { int serviceCount = entry.getCount(); // TODO: ratio constraint if (serviceCount < constraint.getMinCount() || serviceCount > constraint.getMaxCount() || serviceCount > layout.size()) { return false; } } } return true; }
@BeforeClass public static void beforeClass() { constraints = new Constraints( ImmutableMap.<String, ServiceConstraint>of( "master1",
@Test public void testIsValidNodeLayout() { Map<String, ServiceConstraint> serviceConstraints = reactorTemplate.getConstraints().getServiceConstraints(); // test all possible valid node layouts Set<String> masterServices = ImmutableSet.of("namenode", "resourcemanager", "hbasemaster"); assertSatisfiesServiceConstraints("large-mem", "centos6", masterServices, serviceConstraints); assertSatisfiesServiceConstraints("large-mem", "ubuntu12", masterServices, serviceConstraints); Set<String> slaveServices = ImmutableSet.of("datanode", "nodemanager", "regionserver"); assertSatisfiesServiceConstraints("medium", "centos6", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("medium", "ubuntu12", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("large-cpu", "centos6", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("large-cpu", "ubuntu12", slaveServices, serviceConstraints); assertSatisfiesServiceConstraints("small", "centos6", ImmutableSet.of("zookeeper"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("zookeeper"), serviceConstraints); assertSatisfiesServiceConstraints("large", "centos6", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("large", "ubuntu12", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "ubuntu12", ImmutableSet.of("reactor"), serviceConstraints); // there are no image type constraints for reactor so this should pass assertSatisfiesServiceConstraints("medium", "asdf", ImmutableSet.of("reactor"), serviceConstraints); assertSatisfiesServiceConstraints("medium", "centos6", ImmutableSet.of("reactor", "zookeeper"), serviceConstraints); // test hardware type is invalid assertUnsatisfiesServiceConstraints("large", "centos6", masterServices, serviceConstraints); // test image type is invalid assertUnsatisfiesServiceConstraints("large", "asdf", masterServices, serviceConstraints); // test both are invalid assertUnsatisfiesServiceConstraints("large-mem", "ubuntu12", ImmutableSet.of("reactor", "zookeeper"), serviceConstraints); }
@Test public void testNodesMustHaveServices() throws Exception { Set<String> services = ImmutableSet.of("svc1", "svc2", "svc3"); ClusterTemplate template = ClusterTemplate.builder() .setName("simple") .setDescription("all services on all nodes template") .setClusterDefaults(ClusterDefaults.builder().setServices(services).setProvider("joyent").build()) .setCompatibilities(Compatibilities.builder().setServices(services).build()) .setConstraints(new Constraints( ImmutableMap.<String, ServiceConstraint>of("svc1", new ServiceConstraint(null, null, 1, 1)), new LayoutConstraint( ImmutableSet.<Set<String>>of(ImmutableSet.of("svc1", "svc2", "svc3")), ImmutableSet.<Set<String>>of() ), SizeConstraint.EMPTY )).build(); List<NodeLayout> expected = ImmutableList.of( new NodeLayout("small", "centos6", ImmutableSet.of("svc1", "svc2", "svc3")) ); NodeLayoutGenerator nodeLayoutGenerator = new NodeLayoutGenerator(template, services, ImmutableSet.<String>of("small"), ImmutableSet.<String>of("centos6")); List<NodeLayout> actual = nodeLayoutGenerator.generateNodeLayoutPreferences(); Assert.assertEquals(expected, actual); }