@Override protected RMNodeLabelsManager createNodeLabelManager() { RMNodeLabelsManager mgr = new RMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } };
@Override public GetNodesToLabelsResponse getNodeToLabels( GetNodesToLabelsRequest request) throws YarnException, IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); GetNodesToLabelsResponse response = GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabels()); return response; }
/** * This method returns all non labeled resources. * * @param clusterResources * @return Resources */ private Resource getNonLabeledResources(Resource clusterResources) { RMContext rmcontext = scheduler.getRMContext(); RMNodeLabelsManager lm = rmcontext.getNodeLabelManager(); Resource res = lm.getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResources); return res == null ? clusterResources : res; }
public void updateNodeResource(NodeId node, Resource newResource) { deactivateNode(node); activateNode(node, newResource); }
private static Set<NodeId> getNodeIdsForLabel(RMContext rmContext, String label) { label = (label == null || label.trim().isEmpty()) ? RMNodeLabelsManager.NO_LABEL : label; if (label.equals(RMNodeLabelsManager.NO_LABEL)) { // NO_LABEL nodes aren't tracked directly return rmContext.getNodeLabelManager().getNodesWithoutALabel(); } else { Map<String, Set<NodeId>> labelsToNodes = rmContext.getNodeLabelManager().getLabelsToNodes( Collections.singleton(label)); return labelsToNodes.get(label); } }
Map<String, Host> before = cloneNodeMap(ImmutableSet.of(nodeId)); Node nm = getNMInNodeSet(nodeId); if (null != nm) { if (null == nm.labels) { Set<String> savedNodeLabels = getLabelsOnNode(nodeId); removeNodeFromLabels(nodeId, savedNodeLabels); } else { Map<String, Host> after = cloneNodeMap(ImmutableSet.of(nodeId)); updateResourceMappings(before, after); } finally { writeLock.unlock();
clusterNodeLabels.add("z"); nodeLabelManager.addToCluserNodeLabels(clusterNodeLabels); nodeLabelManager.addLabelsToNode(ImmutableMap.of(n1, toSet("x"))); nodeLabelManager.removeFromClusterNodeLabels(toSet("z")); nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(n1, toSet("y"))); if (nodeLabelManager.getNodeLabels().size() > 0) { break; .getClusterNodeLabels().size()); Map<NodeId, Set<String>> nodeLabels = nodeLabelManager.getNodeLabels(); Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size()); Assert.assertTrue(nodeLabels.get(n1).equals(toSet("y"))); .getClusterNodeLabels().size()); nodeLabels = nodeLabelManager.getNodeLabels(); Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size()); Assert.assertTrue(nodeLabels.get(n1).equals(toSet("y"))); rm1.stop();
clusterNodeLabels.add("z"); nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(clusterNodeLabels); nodeLabelManager.addLabelsToNode(ImmutableMap.of(n1, toSet("x"))); nodeLabelManager.removeFromClusterNodeLabels(toSet("z")); nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(n1, toSet("y"))); if (nodeLabelManager.getNodeLabels().size() > 0) { break; .getClusterNodeLabelNames().size()); Map<NodeId, Set<String>> nodeLabels = nodeLabelManager.getNodeLabels(); Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size()); Assert.assertTrue(nodeLabels.get(n1).equals(toSet("y"))); nodeLabelManager.getClusterNodeLabelNames().size()); nodeLabels = nodeLabelManager.getNodeLabels(); Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size()); Assert.assertTrue(nodeLabels.get(n1).equals(toSet("y"))); rm1.stop();
@Test public void testRMHARecoverNodeLabels() throws Exception { // start two RMs, and transit rm1 to active, rm2 to standby startRMs(); // Add labels to rm1 rm1.getRMContext() .getNodeLabelManager() .addToCluserNodeLabels(ImmutableSet.of("a", "b", "c")); Map<NodeId, Set<String>> nodeToLabels = new HashMap<>(); nodeToLabels.put(NodeId.newInstance("host1", 0), ImmutableSet.of("a")); nodeToLabels.put(NodeId.newInstance("host2", 0), ImmutableSet.of("b")); rm1.getRMContext().getNodeLabelManager().replaceLabelsOnNode(nodeToLabels); // Do the failover explicitFailover(); // Check labels in rm2 Assert .assertTrue(rm2.getRMContext().getNodeLabelManager() .getClusterNodeLabels() .containsAll(ImmutableSet.of("a", "b", "c"))); Assert.assertTrue(rm2.getRMContext().getNodeLabelManager() .getNodeLabels().get(NodeId.newInstance("host1", 0)).contains("a")); Assert.assertTrue(rm2.getRMContext().getNodeLabelManager() .getNodeLabels().get(NodeId.newInstance("host2", 0)).contains("b")); } }
@Test(timeout = 60000) public void testcheckRemoveFromClusterNodeLabelsOfQueue() throws Exception { lmgr = new RMNodeLabelsManager(); Configuration conf = new Configuration(); File tempDir = File.createTempFile("nlb", ".tmp"); Configuration withQueueLabels = getConfigurationWithQueueLabels(conf); MockRM rm = initRM(conf); lmgr.addToCluserNodeLabels(toSet(NodeLabel.newInstance("x", false))); lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "x" })); lmgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("x")); rm.stop(); class TestRMLabelManger extends RMNodeLabelsManager { + "on recovery", checkQueueCall); lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "x" })); Assert .assertTrue("checkRemoveFromClusterNodeLabelsOfQueue should be called "
.addToCluserNodeLabels( Arrays.asList(NodeLabel.newInstance("a"), NodeLabel.newInstance("b"), NodeLabel.newInstance("c"))); nodeToLabels.put(NodeId.newInstance("host2", 0), ImmutableSet.of("b")); rm1.getRMContext().getNodeLabelManager().replaceLabelsOnNode(nodeToLabels); .getClusterNodeLabelNames() .containsAll(ImmutableSet.of("a", "b", "c"))); Assert.assertTrue(rm2.getRMContext().getNodeLabelManager() .getNodeLabels().get(NodeId.newInstance("host1", 0)).contains("a")); Assert.assertTrue(rm2.getRMContext().getNodeLabelManager() .getNodeLabels().get(NodeId.newInstance("host2", 0)).contains("b"));
if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { String msg = "User " + callerUGI.getShortUserName() + " not authorized" + " for post to .../nodes/nodeid/replace-labels"; rm.getRMContext().getNodeLabelManager().replaceLabelsOnNode(newLabelsForNode);
@Override public AddToClusterNodeLabelsResponse addToClusterNodeLabels(AddToClusterNodeLabelsRequest request) throws YarnException, IOException { String argName = "addToClusterNodeLabels"; final String msg = "add labels."; UserGroupInformation user = checkAcls(argName); checkRMStatus(user.getShortUserName(), argName, msg); AddToClusterNodeLabelsResponse response = recordFactory.newRecordInstance(AddToClusterNodeLabelsResponse.class); try { rmContext.getNodeLabelManager().addToCluserNodeLabels(request.getNodeLabels()); RMAuditLogger .logSuccess(user.getShortUserName(), argName, "AdminService"); return response; } catch (IOException ioe) { throw logAndWrapException(ioe, user.getShortUserName(), argName, msg); } }
@POST @Path("/add-node-labels") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels, @Context HttpServletRequest hsr) throws Exception { init(); UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); if (callerUGI == null) { String msg = "Unable to obtain user name, user not authenticated for" + " post to .../add-node-labels"; throw new AuthorizationException(msg); } if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { String msg = "User " + callerUGI.getShortUserName() + " not authorized" + " for post to .../add-node-labels "; throw new AuthorizationException(msg); } rm.getRMContext().getNodeLabelManager() .addToCluserNodeLabels(new HashSet<String>( newNodeLabels.getNodeLabels())); return Response.status(Status.OK).build(); }
rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of("x", "y")); Resource resource = Resources.createResource( fail("Should be valid when request labels is a subset of queue labels"); } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y"));
@Test(timeout = 60000) public void testBackwardsCompatableMirror() throws Exception { lmgr = new RMNodeLabelsManager(); Configuration conf = new Configuration(); File tempDir = File.createTempFile("nlb", ".tmp"); Set<String> labelNames = lmgr.getClusterNodeLabelNames(); Map<String, Set<NodeId>> labeledNodes = lmgr.getLabelsToNodes();
@POST @Path(RMWSConsts.REMOVE_NODE_LABELS) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Override public Response removeFromCluserNodeLabels( @QueryParam(RMWSConsts.LABELS) Set<String> oldNodeLabels, @Context HttpServletRequest hsr) throws Exception { UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); initForWritableEndpoints(callerUGI, false); if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { String msg = "User " + callerUGI.getShortUserName() + " not authorized" + " for post to .../remove-node-labels "; throw new AuthorizationException(msg); } try { rm.getRMContext().getNodeLabelManager() .removeFromClusterNodeLabels(new HashSet<String>(oldNodeLabels)); } catch (IOException e) { throw new BadRequestException(e); } return Response.status(Status.OK).build(); }
/** * Gets the labels which are accessible by this queue. If ANY label can be * accessed, put all labels in the set. * @return accessiglbe node labels */ protected final Set<String> getAccessibleLabelSet() { Set<String> nodeLabels = new HashSet<String>(); if (this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { nodeLabels.addAll(labelManager.getClusterNodeLabels()); } else { nodeLabels.addAll(this.getAccessibleNodeLabels()); } nodeLabels.add(RMNodeLabelsManager.NO_LABEL); return nodeLabels; }
clusterNodeLabels.add("y"); nodeLabelManager .addToCluserNodeLabelsWithDefaultExclusivity(clusterNodeLabels); nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(n1, toSet("x"))); nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(n2, toSet("x"))); nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(nihost, toSet("y"))); nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(n1, toSet("x"))); MockRM rm2 = null; for (int i = 0; i < 2; i++) { nodeLabelManager.getLabelsToNodes(toSet("x")); Assert.assertEquals(1, null == labelsToNodes.get("x") ? 0 : labelsToNodes.get("x").size());
labelSet.add("x"); labels.put(node, labelSet); when(lm.getNodeLabels()).thenReturn(labels); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); when(lm.getResourceByLabel(anyString(), any(Resource.class))).thenReturn( Resources.clone(Resource.newInstance(80, 0))); clusterResources.setMemory(100);