@Override public Resource getTotalCapability() { return node.getTotalCapability(); }
public SchedulerNode(RMNode node, boolean usePortForNodeName, Set<String> labels) { this.rmNode = node; this.availableResource = Resources.clone(node.getTotalCapability()); this.totalResourceCapability = Resources.clone(node.getTotalCapability()); if (usePortForNodeName) { nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort(); } else { nodeName = rmNode.getHostName(); } this.labels = ImmutableSet.copyOf(labels); }
public SchedulerNode(RMNode node, boolean usePortForNodeName, Set<String> labels) { this.rmNode = node; this.availableResource = Resources.clone(node.getTotalCapability()); this.totalResourceCapability = Resources.clone(node.getTotalCapability()); if (usePortForNodeName) { nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort(); } else { nodeName = rmNode.getHostName(); } this.labels = ImmutableSet.copyOf(labels); }
private RMNode createNode() { RMNode node = mock(RMNode.class); when(node.getTotalCapability()).thenReturn(Resource.newInstance(8192, 8)); when(node.getHostName()).thenReturn("host.domain.com"); return node; }
public SchedulerNode(RMNode node, boolean usePortForNodeName, Set<String> labels) { this.rmNode = node; this.rmContext = node.getRMContext(); this.unallocatedResource = Resources.clone(node.getTotalCapability()); this.totalResource = Resources.clone(node.getTotalCapability()); if (usePortForNodeName) { nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort(); } else { nodeName = rmNode.getHostName(); } this.labels = ImmutableSet.copyOf(labels); this.lastHeartbeatMonotonicTime = Time.monotonicNow(); }
public void init(String nodeIdStr, int memory, int cores, int dispatchTime, int heartBeatInterval, ResourceManager rm) throws IOException, YarnException { super.init(dispatchTime, dispatchTime + 1000000L * heartBeatInterval, heartBeatInterval); // create resource String rackHostName[] = SLSUtils.getRackHostName(nodeIdStr); this.node = NodeInfo.newNodeInfo(rackHostName[0], rackHostName[1], BuilderUtils.newResource(memory, cores)); this.rm = rm; // init data structures completedContainerList = Collections.synchronizedList(new ArrayList<ContainerId>()); releasedContainerList = Collections.synchronizedList(new ArrayList<ContainerId>()); containerQueue = new DelayQueue<ContainerSimulator>(); amContainerList = Collections.synchronizedList(new ArrayList<ContainerId>()); runningContainers = new ConcurrentHashMap<ContainerId, ContainerSimulator>(); // register NM with RM RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class); req.setNodeId(node.getNodeID()); req.setResource(node.getTotalCapability()); req.setHttpPort(80); RegisterNodeManagerResponse response = rm.getResourceTrackerService() .registerNodeManager(req); masterKey = response.getNMTokenMasterKey(); }
public static FiCaSchedulerNode getMockNode(String host, String rack, int port, int memory, int vcores) { NodeId nodeId = NodeId.newInstance(host, port); RMNode rmNode = mock(RMNode.class); when(rmNode.getNodeID()).thenReturn(nodeId); when(rmNode.getTotalCapability()).thenReturn( Resources.createResource(memory, vcores)); when(rmNode.getNodeAddress()).thenReturn(host+":"+port); when(rmNode.getHostName()).thenReturn(host); when(rmNode.getRackName()).thenReturn(rack); when(rmNode.getState()).thenReturn(NodeState.RUNNING); FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false)); LOG.info("node = " + host + " avail=" + node.getUnallocatedResource()); when(node.getNodeID()).thenReturn(nodeId); return node; }
public static FiCaSchedulerNode getMockNode( String host, String rack, int port, int capability) { NodeId nodeId = mock(NodeId.class); when(nodeId.getHost()).thenReturn(host); when(nodeId.getPort()).thenReturn(port); RMNode rmNode = mock(RMNode.class); when(rmNode.getNodeID()).thenReturn(nodeId); when(rmNode.getTotalCapability()).thenReturn( Resources.createResource(capability, 1)); when(rmNode.getNodeAddress()).thenReturn(host+":"+port); when(rmNode.getHostName()).thenReturn(host); when(rmNode.getRackName()).thenReturn(rack); FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false)); LOG.info("node = " + host + " avail=" + node.getAvailableResource()); return node; }
private void handleNodeUpdates(RMApp app, AllocateResponse allocateResponse) { Map<RMNode, NodeUpdateType> updatedNodes = new HashMap<>(); if(app.pullRMNodeUpdates(updatedNodes) > 0) { List<NodeReport> updatedNodeReports = new ArrayList<>(); for(Map.Entry<RMNode, NodeUpdateType> rmNodeEntry : updatedNodes.entrySet()) { RMNode rmNode = rmNodeEntry.getKey(); SchedulerNodeReport schedulerNodeReport = getScheduler().getNodeReport(rmNode.getNodeID()); Resource used = BuilderUtils.newResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); numContainers = schedulerNodeReport.getNumContainers(); } NodeId nodeId = rmNode.getNodeID(); NodeReport report = BuilderUtils.newNodeReport(nodeId, rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(), rmNodeEntry.getValue()); updatedNodeReports.add(report); } allocateResponse.setUpdatedNodes(updatedNodeReports); } }
private void verifyNodesAfterDecom(MockRM rm, int numNodes, Resource expectedCapability, String expectedVersion) { ConcurrentMap<NodeId, RMNode> inactiveRMNodes = rm.getRMContext().getInactiveRMNodes(); Assert.assertEquals(numNodes, inactiveRMNodes.size()); for (RMNode rmNode : inactiveRMNodes.values()) { Assert.assertEquals(expectedCapability, rmNode.getTotalCapability()); Assert.assertEquals(expectedVersion, rmNode.getNodeManagerVersion()); } }
private void verifyNodesAfterDecom(MockRM rm, int numNodes, Resource expectedCapability, String expectedVersion) { ConcurrentMap<String, RMNode> inactiveRMNodes = rm.getRMContext().getInactiveRMNodes(); Assert.assertEquals(numNodes, inactiveRMNodes.size()); for (RMNode rmNode : inactiveRMNodes.values()) { Assert.assertEquals(expectedCapability, rmNode.getTotalCapability()); Assert.assertEquals(expectedVersion, rmNode.getNodeManagerVersion()); } }
private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); Resource used = BuilderUtils.newResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); numContainers = schedulerNodeReport.getNumContainers(); } NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels()); return report; }
private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); Resource used = BuilderUtils.newResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); numContainers = schedulerNodeReport.getNumContainers(); } NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels()); return report; }
public FifoSchedulerInfo(final ResourceManager rm) { RMContext rmContext = rm.getRMContext(); FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler(); qName = fs.getQueueInfo("", false, false).getQueueName(); QueueInfo qInfo = fs.getQueueInfo(qName, true, true); this.usedCapacity = qInfo.getCurrentCapacity(); this.capacity = qInfo.getCapacity(); this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory(); this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory(); this.qstate = qInfo.getQueueState(); this.numNodes = rmContext.getRMNodes().size(); this.usedNodeCapacity = 0; this.availNodeCapacity = 0; this.totalNodeCapacity = 0; this.numContainers = 0; for (RMNode ni : rmContext.getRMNodes().values()) { SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID()); this.usedNodeCapacity += report.getUsedResource().getMemory(); this.availNodeCapacity += report.getAvailableResource().getMemory(); this.totalNodeCapacity += ni.getTotalCapability().getMemory(); this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers(); } }
public FifoSchedulerInfo(final ResourceManager rm) { RMContext rmContext = rm.getRMContext(); FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler(); qName = fs.getQueueInfo("", false, false).getQueueName(); QueueInfo qInfo = fs.getQueueInfo(qName, true, true); this.usedCapacity = qInfo.getCurrentCapacity(); this.capacity = qInfo.getCapacity(); this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory(); this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory(); this.qstate = qInfo.getQueueState(); this.numNodes = rmContext.getRMNodes().size(); this.usedNodeCapacity = 0; this.availNodeCapacity = 0; this.totalNodeCapacity = 0; this.numContainers = 0; for (RMNode ni : rmContext.getRMNodes().values()) { SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID()); this.usedNodeCapacity += report.getUsedResource().getMemory(); this.availNodeCapacity += report.getAvailableResource().getMemory(); this.totalNodeCapacity += ni.getTotalCapability().getMemory(); this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers(); } }
public FifoSchedulerInfo(final ResourceManager rm) { RMContext rmContext = rm.getRMContext(); FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler(); qName = fs.getQueueInfo("", false, false).getQueueName(); QueueInfo qInfo = fs.getQueueInfo(qName, true, true); this.usedCapacity = qInfo.getCurrentCapacity(); this.capacity = qInfo.getCapacity(); this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemorySize(); this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemorySize(); this.qstate = qInfo.getQueueState(); this.numNodes = rmContext.getRMNodes().size(); this.usedNodeCapacity = 0; this.availNodeCapacity = 0; this.totalNodeCapacity = 0; this.numContainers = 0; for (RMNode ni : rmContext.getRMNodes().values()) { SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID()); this.usedNodeCapacity += report.getUsedResource().getMemorySize(); this.availNodeCapacity += report.getAvailableResource().getMemorySize(); this.totalNodeCapacity += ni.getTotalCapability().getMemorySize(); this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers(); } }
/** * Whether this app has containers requests that could be satisfied on the * given node, if the node had full space. */ public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) { ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY); ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName()); ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName()); return // There must be outstanding requests at the given priority: anyRequest != null && anyRequest.getNumContainers() > 0 && // If locality relaxation is turned off at *-level, there must be a // non-zero request for the node's rack: (anyRequest.getRelaxLocality() || (rackRequest != null && rackRequest.getNumContainers() > 0)) && // If locality relaxation is turned off at rack-level, there must be a // non-zero request at the node: (rackRequest == null || rackRequest.getRelaxLocality() || (nodeRequest != null && nodeRequest.getNumContainers() > 0)) && // The requested container must be able to fit on the node: Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, anyRequest.getCapability(), node.getRMNode().getTotalCapability()); }
/** * Whether this app has containers requests that could be satisfied on the * given node, if the node had full space. */ public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) { ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY); ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName()); ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName()); return // There must be outstanding requests at the given priority: anyRequest != null && anyRequest.getNumContainers() > 0 && // If locality relaxation is turned off at *-level, there must be a // non-zero request for the node's rack: (anyRequest.getRelaxLocality() || (rackRequest != null && rackRequest.getNumContainers() > 0)) && // If locality relaxation is turned off at rack-level, there must be a // non-zero request at the node: (rackRequest == null || rackRequest.getRelaxLocality() || (nodeRequest != null && nodeRequest.getNumContainers() > 0)) && // The requested container must be able to fit on the node: Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, anyRequest.getCapability(), node.getRMNode().getTotalCapability()); }
private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); Resource used = BuilderUtils.newResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); numContainers = schedulerNodeReport.getNumContainers(); } Set<NodeAttribute> attrs = rmNode.getAllNodeAttributes(); NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(), rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(), null, attrs); return report; }
Resource resource = ni.getTotalCapability(); Assert.assertEquals("<memory:5120, vCores:5>", resource.toString()); Resource resourceAfter = niAfter.getTotalCapability(); Assert.assertEquals("<memory:4096, vCores:4>", resourceAfter.toString());