@Test(timeout = 5000) public void testGetLabelsOnNodesWhenNodeActiveDeactive() throws Exception { mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of( toNodeId("n1"), toSet("p2"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); mgr.activateNode(toNodeId("n1:1"), SMALL_RESOURCE); assertCollectionEquals(toSet("p1"), mgr.getNodeLabels().get(toNodeId("n1:1"))); mgr.deactivateNode(toNodeId("n1:1")); assertCollectionEquals(toSet("p1"), mgr.getNodeLabels().get(toNodeId("n1:1"))); assertCollectionEquals(toSet("p2"), mgr.getNodeLabels().get(toNodeId("n1"))); mgr.activateNode(toNodeId("n1:2"), SMALL_RESOURCE); assertCollectionEquals(toSet("p2"), mgr.getNodeLabels().get(toNodeId("n1:2"))); mgr.deactivateNode(toNodeId("n1:2")); Assert.assertNull(mgr.getNodeLabels().get(toNodeId("n1:2"))); assertCollectionEquals(toSet("p2"), mgr.getNodeLabels().get(toNodeId("n1"))); mgr.activateNode(toNodeId("n1:2"), SMALL_RESOURCE); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p3"))); assertCollectionEquals(toSet("p3"), mgr.getNodeLabels().get(toNodeId("n1:2")));
@Test(timeout = 5000) public void testLabelsToNodesOnNodeActiveDeactive() throws Exception { // Activate a node without assigning any labels mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); Assert.assertTrue(mgr.getLabelsToNodes().isEmpty()); assertLabelsToNodesEquals( mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); // Add labels and replace labels on node mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); // p1 -> n1, n1:1 Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size()); assertLabelsToNodesEquals( mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); // Activate a node for which host to label mapping exists mgr.activateNode(NodeId.newInstance("n1", 2), Resource.newInstance(10, 0)); // p1 -> n1, n1:1, n1:2 Assert.assertEquals(3, mgr.getLabelsToNodes().get("p1").size()); assertLabelsToNodesEquals( mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); // Deactivate a node. n1:1 will be removed from the map mgr.deactivateNode(NodeId.newInstance("n1", 1)); // p1 -> n1, n1:2 Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size()); assertLabelsToNodesEquals( mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); }
@Test(timeout = 5000) public void testPullRMNodeLabelsInfo() throws IOException { mgr.addToCluserNodeLabels(toSet("x", "y", "z")); mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0)); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"), toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y"))); // x, y, z and "" List<NodeLabel> infos = mgr.pullRMNodeLabelsInfo(); Assert.assertEquals(4, infos.size()); checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20); checkNodeLabelInfo(infos, "x", 2, 20); checkNodeLabelInfo(infos, "y", 1, 10); checkNodeLabelInfo(infos, "z", 0, 0); }
@Test(timeout = 5000) public void testRemoveLabelsFromNode() throws Exception { mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); // active one NM to n1:1 mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE); try { mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); Assert.fail("removeLabelsFromNode should trigger IOException"); } catch (IOException e) { } mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); try { mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); } catch (IOException e) { Assert.fail("IOException from removeLabelsFromNode " + e); } }
conf.set(YarnConfiguration.RM_SCHEDULER, "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); Configuration withQueueLabels = getConfigurationWithQueueLabels(conf); MockRM rm = initRM(conf); lmgr.addToCluserNodeLabels(toSet(NodeLabel.newInstance("x", false))); lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "x" })); lmgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("x")); rm.stop(); class TestRMLabelManger extends RMNodeLabelsManager { MockRM rm2 = initRM(withQueueLabels); Assert.assertFalse( "checkRemoveFromClusterNodeLabelsOfQueue should not be called"
conf.set(YarnConfiguration.RM_SCHEDULER, "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); Configuration withQueueLabels = getConfigurationWithQueueLabels(conf); MockRM rm = initRM(withQueueLabels); Set<String> labelNames = lmgr.getClusterNodeLabelNames(); Map<String, Set<NodeId>> labeledNodes = lmgr.getLabelsToNodes();
@Test(timeout = 5000) public void testRemoveLabelsFromNode() throws Exception { mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); // active one NM to n1:1 mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE); try { mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); Assert.fail("removeLabelsFromNode should trigger IOException"); } catch (IOException e) { } mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); try { mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); } catch (IOException e) { Assert.fail("IOException from removeLabelsFromNode " + e); } }
@Test(timeout=5000) public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException { // active two NM to n1, one large and one small mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE); // check resource of no label, it should be small * 4 Assert.assertEquals( mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), Resources.multiply(SMALL_RESOURCE, 4)); // change two of these nodes to p1, check resource of no_label and P1 mgr.addToCluserNodeLabels(toSet("p1")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n1:2"), toSet("p1"))); // check resource Assert.assertEquals( mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), Resources.multiply(SMALL_RESOURCE, 2)); Assert.assertEquals( mgr.getResourceByLabel("p1", null), Resources.multiply(SMALL_RESOURCE, 2)); }
@Test(timeout = 5000) public void testLabelsToNodesOnNodeActiveDeactive() throws Exception { // Activate a node without assigning any labels mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); Assert.assertTrue(mgr.getLabelsToNodes().isEmpty()); assertLabelsToNodesEquals( mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); // Add labels and replace labels on node mgr.addToCluserNodeLabels(toSet("p1")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); // p1 -> n1, n1:1 Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size()); assertLabelsToNodesEquals( mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); // Activate a node for which host to label mapping exists mgr.activateNode(NodeId.newInstance("n1", 2), Resource.newInstance(10, 0)); // p1 -> n1, n1:1, n1:2 Assert.assertEquals(3, mgr.getLabelsToNodes().get("p1").size()); assertLabelsToNodesEquals( mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); // Deactivate a node. n1:1 will be removed from the map mgr.deactivateNode(NodeId.newInstance("n1", 1)); // p1 -> n1, n1:2 Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size()); assertLabelsToNodesEquals( mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); }
@Test(timeout = 5000) public void testGetLabelsOnNodesWhenNodeActiveDeactive() throws Exception { mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of( toNodeId("n1"), toSet("p2"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); mgr.activateNode(toNodeId("n1:1"), SMALL_RESOURCE); assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1:1")), toSet("p1")); mgr.deactivateNode(toNodeId("n1:1")); assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1:1")), toSet("p1")); assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1")), toSet("p2")); mgr.activateNode(toNodeId("n1:2"), SMALL_RESOURCE); assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1:2")), toSet("p2")); mgr.deactivateNode(toNodeId("n1:2")); Assert.assertNull(mgr.getNodeLabels().get(toNodeId("n1:2"))); assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1")), toSet("p2")); mgr.activateNode(toNodeId("n1:2"), SMALL_RESOURCE); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p3"))); assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1:2")), toSet("p3"));
@Test(timeout = 5000) public void testPullRMNodeLabelsInfo() throws IOException { mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("x", "y", "z")); mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0)); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"), toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y"))); // x, y, z and "" List<RMNodeLabel> infos = mgr.pullRMNodeLabelsInfo(); Assert.assertEquals(4, infos.size()); checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20); checkNodeLabelInfo(infos, "x", 2, 20); checkNodeLabelInfo(infos, "y", 1, 10); checkNodeLabelInfo(infos, "z", 0, 0); }
@SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 5000) public void testGetLabelResource() throws Exception { mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p2"))); Assert.assertEquals(mgr.getResourceByLabel("p1", null), EMPTY_RESOURCE); Assert.assertEquals(mgr.getResourceByLabel("p2", null), mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4", "p5", "p6")); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n4"), toSet("p1"), toNodeId("n5"), toSet("p2"), toNodeId("n6"), toSet("p3"), toNodeId("n7"), toSet("p4"), toNodeId("n8"), toSet("p5"))); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n4"), toSet("p2"), toNodeId("n7"), RMNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n5"), toSet("p1"), toNodeId("n8"), RMNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n9"), toSet("p1")));
mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n2", 1), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n3", 1), SMALL_RESOURCE); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n2:1"), toSet("p2"), toNodeId("n3"), toSet("p3"))); assertTrue("Event should be sent when there is change in labels", schedEventsHandler.receivedEvent); schedEventsHandler.updatedNodeToLabels.size()); ImmutableMap<NodeId, Set<String>> modifiedMap = ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n2:1"), toSet("p2"), toNodeId("n3:1"), toSet("p3")); assertEquals("Node label mapping is not matching", modifiedMap, schedEventsHandler.updatedNodeToLabels); schedEventsHandler.receivedEvent = false; mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); assertFalse("No event should be sent when there is no change in labels", schedEventsHandler.receivedEvent); schedEventsHandler.receivedEvent = false; mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2:1"), toSet("p1"), toNodeId("n3"), toSet("p3"))); assertTrue("Event should be sent when there is change in labels", schedEventsHandler.receivedEvent); assertEquals("Single node label mapping modified", 1, schedEventsHandler.updatedNodeToLabels.size());
@Test(timeout=5000) public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException { // active two NM to n1, one large and one small mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE); // check resource of no label, it should be small * 4 Assert.assertEquals( mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), Resources.multiply(SMALL_RESOURCE, 4)); // change two of these nodes to p1, check resource of no_label and P1 mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n1:2"), toSet("p1"))); // check resource Assert.assertEquals( mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), Resources.multiply(SMALL_RESOURCE, 2)); Assert.assertEquals( mgr.getResourceByLabel("p1", null), Resources.multiply(SMALL_RESOURCE, 2)); }
@SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 5000) public void testGetLabelResource() throws Exception { mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p2"))); Assert.assertEquals(mgr.getResourceByLabel("p1", null), EMPTY_RESOURCE); Assert.assertEquals(mgr.getResourceByLabel("p2", null), mgr.addToCluserNodeLabels(toSet("p4", "p5", "p6")); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n4"), toSet("p1"), toNodeId("n5"), toSet("p2"), toNodeId("n6"), toSet("p3"), toNodeId("n7"), toSet("p4"), toNodeId("n8"), toSet("p5"))); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n4"), toSet("p2"), toNodeId("n7"), RMNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n5"), toSet("p1"), toNodeId("n8"), RMNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n9"), toSet("p1")));
@Test(timeout = 5000) public void testGetLabelResourceWhenNodeActiveDeactive() throws Exception { mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); mgr.addToCluserNodeLabels(toSet("p1", "p4")); Assert.assertEquals(mgr.getResourceByLabel("p1", null), Resources.add(SMALL_RESOURCE, LARGE_NODE));
@Test(timeout = 5000) public void testGetLabelResourceWhenNodeActiveDeactive() throws Exception { mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p4")); Assert.assertEquals(mgr.getResourceByLabel("p1", null), Resources.add(SMALL_RESOURCE, LARGE_NODE));
mgr.addToCluserNodeLabels(toSet("red", "blue", "yellow")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"), toSet("red"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"), toSet("blue"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host3"), toSet("yellow"))); Set<String> q1Label = toSet("red", "blue"); Set<String> q2Label = toSet("blue", "yellow"); Set<String> q3Label = toSet("yellow"); Set<String> q4Label = RMNodeLabelsManager.EMPTY_STRING_SET; Set<String> q5Label = toSet(RMNodeLabelsManager.ANY); mgr.getQueueResource("Q5", q5Label, clusterResource)); mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("host2"), toSet("blue"))); q1Label = toSet("blue"); q2Label = toSet("blue", "red"); q3Label = toSet("red"); q4Label = RMNodeLabelsManager.EMPTY_STRING_SET; q5Label = toSet(RMNodeLabelsManager.ANY);
mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("red", "blue", "yellow")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"), toSet("red"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"), toSet("blue"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host3"), toSet("yellow"))); Set<String> q1Label = toSet("red", "blue"); Set<String> q2Label = toSet("blue", "yellow"); Set<String> q3Label = toSet("yellow"); Set<String> q4Label = RMNodeLabelsManager.EMPTY_STRING_SET; Set<String> q5Label = toSet(RMNodeLabelsManager.ANY); mgr.getQueueResource("Q5", q5Label, clusterResource)); mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("host2"), toSet("blue"))); q1Label = toSet("blue"); q2Label = toSet("blue", "red"); q3Label = toSet("red"); q4Label = RMNodeLabelsManager.EMPTY_STRING_SET; q5Label = toSet(RMNodeLabelsManager.ANY);