private RMNode getRmNode(NodeId nodeId) { RMNode rmNode = this.rmContext.getRMNodes().get(nodeId); if (rmNode == null) { rmNode = this.rmContext.getInactiveRMNodes().get(nodeId); } return rmNode; }
public static List<RMNode> queryRMNodes(RMContext context, EnumSet<NodeState> acceptedStates) { // nodes contains nodes that are NEW, RUNNING OR UNHEALTHY ArrayList<RMNode> results = new ArrayList<RMNode>(); if (acceptedStates.contains(NodeState.NEW) || acceptedStates.contains(NodeState.RUNNING) || acceptedStates.contains(NodeState.UNHEALTHY)) { for (RMNode rmNode : context.getRMNodes().values()) { if (acceptedStates.contains(rmNode.getState())) { results.add(rmNode); } } } // inactiveNodes contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED if (acceptedStates.contains(NodeState.DECOMMISSIONED) || acceptedStates.contains(NodeState.LOST) || acceptedStates.contains(NodeState.REBOOTED)) { for (RMNode rmNode : context.getInactiveRMNodes().values()) { if (acceptedStates.contains(rmNode.getState())) { results.add(rmNode); } } } return results; }
/** * @param node * @return true if valid else false; */ private boolean validateForInvalidNode(String node, boolean failOnUnknownNodes) { if (!failOnUnknownNodes) { return true; } // both active and inactive nodes are recognized as known nodes boolean isKnown = rm.getRMContext().getRMNodes().keySet().stream() .anyMatch(activeNode -> activeNode.getHost().equals(node)); if (!isKnown) { isKnown = rm.getRMContext().getInactiveRMNodes().keySet().stream() .anyMatch(inactiveNode -> inactiveNode.getHost().equals(node)); } return isKnown; } }
private void updateInactiveNodes() { long now = Time.monotonicNow(); for(Entry<NodeId, RMNode> entry : rmContext.getInactiveRMNodes().entrySet()) { NodeId nodeId = entry.getKey(); RMNode rmNode = entry.getValue(); if (isUntrackedNode(nodeId.getHost()) && rmNode.getUntrackedTimeStamp() == 0) { rmNode.setUntrackedTimeStamp(now); } } }
private RMNode getRMNode(NodeId nodeId) { RMNode node = getRMContext().getRMNodes().get(nodeId); if (node == null) { node = getRMContext().getInactiveRMNodes().get(nodeId); } return node; }
@GET @Path("/nodes/{nodeId}") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public NodeInfo getNode(@PathParam("nodeId") String nodeId) { init(); if (nodeId == null || nodeId.isEmpty()) { throw new NotFoundException("nodeId, " + nodeId + ", is empty or null"); } ResourceScheduler sched = this.rm.getResourceScheduler(); if (sched == null) { throw new NotFoundException("Null ResourceScheduler instance"); } NodeId nid = ConverterUtils.toNodeId(nodeId); RMNode ni = this.rm.getRMContext().getRMNodes().get(nid); boolean isInactive = false; if (ni == null) { ni = this.rm.getRMContext().getInactiveRMNodes().get(nid.getHost()); if (ni == null) { throw new NotFoundException("nodeId, " + nodeId + ", is not found"); } isInactive = true; } NodeInfo nodeInfo = new NodeInfo(ni, sched); if (isInactive) { nodeInfo.setNodeHTTPAddress(EMPTY); } return nodeInfo; }
private void setDecomissionedNMs() { Set<String> excludeList = hostsReader.getExcludedHosts(); for (final String host : excludeList) { NodeId nodeId = createUnknownNodeId(host); RMNodeImpl rmNode = new RMNodeImpl(nodeId, rmContext, host, -1, -1, new UnknownNode(host), Resource.newInstance(0, 0), "unknown"); rmContext.getInactiveRMNodes().put(nodeId, rmNode); rmNode.handle(new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION)); } }
private void setDecomissionedNMs() { Set<String> excludeList = hostsReader.getExcludedHosts(); for (final String host : excludeList) { NodeId nodeId = createUnknownNodeId(host); RMNodeImpl rmNode = new RMNodeImpl(nodeId, rmContext, host, -1, -1, new UnknownNode(host), null, null); rmContext.getInactiveRMNodes().put(nodeId.getHost(), rmNode); rmNode.handle(new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION)); } }
/** * Put a node in deactivated (decommissioned or shutdown) status. * @param rmNode * @param finalState */ public static void deactivateNode(RMNodeImpl rmNode, NodeState finalState) { if (rmNode.getNodeID().getPort() == -1) { rmNode.updateMetricsForDeactivatedNode(rmNode.getState(), finalState); return; } reportNodeUnusable(rmNode, finalState); // Deactivate the node rmNode.context.getRMNodes().remove(rmNode.nodeId); LOG.info("Deactivating Node " + rmNode.nodeId + " as it is now " + finalState); rmNode.context.getInactiveRMNodes().put(rmNode.nodeId, rmNode); if (rmNode.context.getNodesListManager().isUntrackedNode(rmNode.hostName)) { rmNode.setUntrackedTimeStamp(Time.monotonicNow()); } }
private void setDecomissionedNMs() { Set<String> excludeList = hostsReader.getExcludedHosts(); for (final String host : excludeList) { NodeId nodeId = createUnknownNodeId(host); RMNodeImpl rmNode = new RMNodeImpl(nodeId, rmContext, host, -1, -1, new UnknownNode(host), Resource.newInstance(0, 0), "unknown"); rmContext.getInactiveRMNodes().put(nodeId.getHost(), rmNode); rmNode.handle(new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION)); } }
@Override public void transition(RMNodeImpl rmNode, RMNodeEvent event) { //check for UnknownNodeId if (rmNode.getNodeID().getPort() == -1) { rmNode.updateMetricsForDeactivatedNode(rmNode.getState(), finalState); return; } // Inform the scheduler rmNode.nodeUpdateQueue.clear(); // If the current state is NodeState.UNHEALTHY // Then node is already been removed from the // Scheduler NodeState initialState = rmNode.getState(); if (!initialState.equals(NodeState.UNHEALTHY)) { rmNode.context.getDispatcher().getEventHandler() .handle(new NodeRemovedSchedulerEvent(rmNode)); } rmNode.context.getDispatcher().getEventHandler().handle( new NodesListManagerEvent( NodesListManagerEventType.NODE_UNUSABLE, rmNode)); // Deactivate the node rmNode.context.getRMNodes().remove(rmNode.nodeId); LOG.info("Deactivating Node " + rmNode.nodeId + " as it is now " + finalState); rmNode.context.getInactiveRMNodes().put(rmNode.nodeId.getHost(), rmNode); //Update the metrics rmNode.updateMetricsForDeactivatedNode(initialState, finalState); } }
@Override public void transition(RMNodeImpl rmNode, RMNodeEvent event) { //check for UnknownNodeId if (rmNode.getNodeID().getPort() == -1) { rmNode.updateMetricsForDeactivatedNode(rmNode.getState(), finalState); return; } // Inform the scheduler rmNode.nodeUpdateQueue.clear(); // If the current state is NodeState.UNHEALTHY // Then node is already been removed from the // Scheduler NodeState initialState = rmNode.getState(); if (!initialState.equals(NodeState.UNHEALTHY)) { rmNode.context.getDispatcher().getEventHandler() .handle(new NodeRemovedSchedulerEvent(rmNode)); } rmNode.context.getDispatcher().getEventHandler().handle( new NodesListManagerEvent( NodesListManagerEventType.NODE_UNUSABLE, rmNode)); // Deactivate the node rmNode.context.getRMNodes().remove(rmNode.nodeId); LOG.info("Deactivating Node " + rmNode.nodeId + " as it is now " + finalState); rmNode.context.getInactiveRMNodes().put(rmNode.nodeId.getHost(), rmNode); //Update the metrics rmNode.updateMetricsForDeactivatedNode(initialState, finalState); } }
private void verifyNodesAfterDecom(MockRM rm, int numNodes, Resource expectedCapability, String expectedVersion) { ConcurrentMap<NodeId, RMNode> inactiveRMNodes = rm.getRMContext().getInactiveRMNodes(); Assert.assertEquals(numNodes, inactiveRMNodes.size()); for (RMNode rmNode : inactiveRMNodes.values()) { Assert.assertEquals(expectedCapability, rmNode.getTotalCapability()); Assert.assertEquals(expectedVersion, rmNode.getNodeManagerVersion()); } }
private void verifyNodesAfterDecom(MockRM rm, int numNodes, Resource expectedCapability, String expectedVersion) { ConcurrentMap<String, RMNode> inactiveRMNodes = rm.getRMContext().getInactiveRMNodes(); Assert.assertEquals(numNodes, inactiveRMNodes.size()); for (RMNode rmNode : inactiveRMNodes.values()) { Assert.assertEquals(expectedCapability, rmNode.getTotalCapability()); Assert.assertEquals(expectedVersion, rmNode.getNodeManagerVersion()); } }
@Test public void testUnknownNodeId() { NodeId nodeId = NodesListManager.createUnknownNodeId("host1"); RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null); rmContext.getInactiveRMNodes().putIfAbsent(nodeId,node); node.handle( new RMNodeEvent(node.getNodeID(), RMNodeEventType.DECOMMISSION)); Assert.assertNull( "Must be null as there is no NODE_UNUSABLE update", nodesListManagerEvent); }
private void setActiveAndInactiveNodes(ResourceManager resourceManager) { Map<NodeId, RMNode> rmNodes = resourceManager.getRMContext().getRMNodes(); rmNodes.put(NodeId.newInstance("host1", 1111), new RMNodeImpl(null, resourceManager.getRMContext(), "host1", 0, 0, null, null, null)); rmNodes.put(NodeId.newInstance("host2", 2222), new RMNodeImpl(null, resourceManager.getRMContext(), "host2", 0, 0, null, null, null)); rmNodes.put(NodeId.newInstance("host3", 3333), new RMNodeImpl(null, resourceManager.getRMContext(), "host3", 0, 0, null, null, null)); Map<NodeId, RMNode> rmInactiveNodes = resourceManager.getRMContext().getInactiveRMNodes(); rmInactiveNodes.put(NodeId.newInstance("host4", 4444), new RMNodeImpl(null, resourceManager.getRMContext(), "host4", 0, 0, null, null, null)); } }
@Test public void testUnknownNodeId() { NodeId nodeId = NodesListManager.createUnknownNodeId("host1"); RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null); rmContext.getInactiveRMNodes().putIfAbsent(nodeId.getHost(),node); node.handle( new RMNodeEvent(node.getNodeID(), RMNodeEventType.DECOMMISSION)); Assert.assertNull( "Must be null as there is no NODE_UNUSABLE update", nodesListManagerEvent); }
Assert.assertTrue("Node " + nm1.getNodeId().getHost() + " should be Decommissioned", rm.getRMContext() .getInactiveRMNodes().get(nm1.getNodeId()).getState() == NodeState .DECOMMISSIONED); writeToHostsFile(excludeHostFile, ""); Assert.assertTrue("Node " + nm1.getNodeId().getHost() + " should be Decommissioned", rm.getRMContext() .getInactiveRMNodes().get(nm1.getNodeId()).getState() == NodeState .DECOMMISSIONED); rm.stop();
@Test public void testSingleNodeQueryStateLost() throws JSONException, Exception { WebResource r = resource(); getRunningRMNode("h1", 1234, 5120); RMNode rmnode2 = getRunningRMNode("h2", 1234, 5121); sendLostEvent(rmnode2); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("nodes").path("h2:1234").accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); JSONObject info = json.getJSONObject("node"); String id = info.get("id").toString(); assertEquals("Incorrect Node Information.", "h2:1234", id); RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(rmnode2.getNodeID()); WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "", info.getString("nodeHTTPAddress")); if (rmNode != null) { WebServicesTestUtils.checkStringMatch("state", rmNode.getState().toString(), info.getString("state")); } }
@Test public void testSingleNodeQueryStateLost() throws JSONException, Exception { WebResource r = resource(); MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm2 = rm.registerNode("h2:1234", 5120); rm.sendNodeStarted(nm1); rm.sendNodeStarted(nm2); rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); rm.NMwaitForState(nm2.getNodeId(), NodeState.RUNNING); rm.sendNodeLost(nm1); rm.sendNodeLost(nm2); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("nodes").path("h2:1234").accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); JSONObject info = json.getJSONObject("node"); String id = info.get("id").toString(); assertEquals("Incorrect Node Information.", "h2:1234", id); RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get("h2"); WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "", info.getString("nodeHTTPAddress")); WebServicesTestUtils.checkStringMatch("state", rmNode.getState().toString(), info.getString("state")); }