public void addNode(N node) { writeLock.lock(); try { nodes.put(node.getNodeID(), node); nodeNameToNodeMap.put(node.getNodeName(), node); List<N> nodesPerLabels = nodesPerLabel.get(node.getPartition()); if (nodesPerLabels == null) { nodesPerLabels = new ArrayList<N>(); } nodesPerLabels.add(node); // Update new set of nodes for given partition. nodesPerLabel.put(node.getPartition(), nodesPerLabels); // Update nodes per rack as well String rackName = node.getRackName(); List<N> nodesList = nodesPerRack.get(rackName); if (nodesList == null) { nodesList = new ArrayList<>(); nodesPerRack.put(rackName, nodesList); } nodesList.add(node); // Update cluster capacity Resources.addTo(clusterCapacity, node.getTotalResource()); staleClusterCapacity = Resources.clone(clusterCapacity); // Update maximumAllocation updateMaxResources(node, true); } finally { writeLock.unlock(); } }
public SchedulerNodeReport(SchedulerNode node) { this.used = node.getUsedResource(); this.avail = node.getAvailableResource(); this.num = node.getNumContainers(); }
/** * The Scheduler has allocated containers on this node to the given * application. * @param rmContainer Allocated container */ public void allocateContainer(RMContainer rmContainer) { allocateContainer(rmContainer, false); }
public SchedulerNodeReport(SchedulerNode node) { this.used = node.getAllocatedResource(); this.avail = node.getUnallocatedResource(); this.num = node.getNumContainers(); }
public SimpleCandidateNodeSet(N node) { if (null != node) { // Only one node in the initial CandidateNodeSet this.map = ImmutableMap.of(node.getNodeID(), node); this.partition = node.getPartition(); } else { this.map = Collections.emptyMap(); this.partition = NodeLabel.DEFAULT_NODE_LABEL_PARTITION; } }
private void clearReservation(SchedulerNode node) { String rackName = node.getRackName() == null ? "NULL" : node.getRackName(); try { writeLock.lock(); Set<String> rackReservations = reservations.get(rackName); if (rackReservations != null) { rackReservations.remove(node.getNodeName()); } } finally { writeLock.unlock(); } }
private SchedulerNode newSchedulerNode(String hostname, String rackName, NodeId nodeId) { SchedulerNode node = mock(SchedulerNode.class); when(node.getNodeName()).thenReturn(hostname); when(node.getRackName()).thenReturn(rackName); when(node.getNodeID()).thenReturn(nodeId); return node; }
"SchedulerNode#toString is not in expected format", schedulerNode1 .toString().contains(schedulerNode1.getUnallocatedResource().toString())); assertTrue( "SchedulerNode#toString is not in expected format", schedulerNode1 .toString().contains(schedulerNode1.getAllocatedResource().toString())); Resource.newInstance(nm1.getMemory(), nm1.getvCores()); assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId())); assertTrue(schedulerNode1.isValidContainer(runningContainer .getContainerId())); assertFalse(schedulerNode1.isValidContainer(completedContainer .getContainerId())); assertEquals(2, schedulerNode1.getNumContainers()); schedulerNode1.getUnallocatedResource()); assertEquals(usedResources, schedulerNode1.getAllocatedResource()); Resource availableResources = Resources.subtract(nmResource, usedResources);
"SchedulerNode#toString is not in expected format", schedulerNode1 .toString().contains(schedulerNode1.getAvailableResource().toString())); assertTrue( "SchedulerNode#toString is not in expected format", schedulerNode1 .toString().contains(schedulerNode1.getUsedResource().toString())); Resource.newInstance(nm1.getMemory(), nm1.getvCores()); assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId())); assertTrue(schedulerNode1.isValidContainer(runningContainer .getContainerId())); assertFalse(schedulerNode1.isValidContainer(completedContainer .getContainerId())); assertEquals(2, schedulerNode1.getNumContainers()); schedulerNode1.getAvailableResource()); assertEquals(usedResources, schedulerNode1.getUsedResource()); Resource availableResources = Resources.subtract(nmResource, usedResources);
/** * Release an allocated container on this node. * * @param container * container to be released */ public synchronized void releaseContainer(Container container) { if (!isValidContainer(container.getId())) { LOG.error("Invalid container released " + container); return; } /* remove the containers from the nodemanger */ if (null != launchedContainers.remove(container.getId())) { updateResource(container); } LOG.info("Released container " + container.getId() + " of capacity " + container.getResource() + " on host " + rmNode.getNodeAddress() + ", which currently has " + numContainers + " containers, " + getUsedResource() + " used and " + getAvailableResource() + " available" + ", release resources=" + true); }
/** * Process resource update on a node. */ public synchronized void updateNodeResource(RMNode nm, ResourceOption resourceOption) { SchedulerNode node = getSchedulerNode(nm.getNodeID()); Resource newResource = resourceOption.getResource(); Resource oldResource = node.getTotalResource(); if(!oldResource.equals(newResource)) { // Log resource change LOG.info("Update resource on node: " + node.getNodeName() + " from: " + oldResource + ", to: " + newResource); nodes.remove(nm.getNodeID()); updateMaximumAllocation(node, false); // update resource to node node.setTotalResource(newResource); nodes.put(nm.getNodeID(), (N)node); updateMaximumAllocation(node, true); // update resource to clusterResource Resources.subtractFrom(clusterResource, oldResource); Resources.addTo(clusterResource, newResource); } else { // Log resource change LOG.warn("Update resource on node: " + node.getNodeName() + " with the same resource: " + newResource); } }
.getSchedulerNode(nm_0.getNodeId()).getAllocatedResource(); Assert.assertEquals(usedResource.getMemorySize(), 0); Assert.assertEquals(usedResource.getVirtualCores(), 0); .getSchedulerNode(nm_0.getNodeId()).getTotalResource(); Assert.assertEquals(totalResource.getMemorySize(), 0 * GB); Assert.assertEquals(totalResource.getVirtualCores(), 0); .getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource(); Assert.assertEquals(availableResource.getMemorySize(), 0); Assert.assertEquals(availableResource.getVirtualCores(), 0);
Assert.assertEquals(1, sn1.getNumContainers()); Assert.assertEquals(1, sn2.getNumContainers()); while (sn1.getCopiedListOfRunningContainers().size() == 1) { Thread.sleep(100); while (sn1.getCopiedListOfRunningContainers().size() == 0) { nm1.nodeHeartbeat(true); Thread.sleep(100); Container container = Container.newInstance( ContainerId.newContainerId(am.getApplicationAttemptId(), 3), sn2.getNodeID(), sn2.getHttpAddress(), reservedResource, Priority.newInstance(0), null); RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey .create(ResourceRequest .newInstance(Priority.newInstance(0), "*", reservedResource, 1)), am.getApplicationAttemptId(), sn2.getNodeID(), "user", rm.getRMContext()); SchedulerContainer reservedContainer = new SchedulerContainer(schedulerApp, scheduler.getNode(sn2.getNodeID()), rmContainer, "", false); ContainerAllocationProposal reservedForAttempt1Proposal = scheduler.tryCommit(scheduler.getClusterResource(), request, true); Assert.assertNull("Outdated proposal should not be accepted!", sn2.getReservedContainer());
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080)); when(mockNode1.getAvailableResource()).thenReturn(emptyResource); when(mockNode1.getTotalResource()).thenReturn(fullResource1); when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081)); when(mockNode2.getAvailableResource()).thenReturn(emptyResource); when(mockNode2.getTotalResource()).thenReturn(fullResource2); scheduler.nodes.put(mockNode1.getNodeID(), mockNode1); scheduler.updateMaximumAllocation(mockNode1, true); verifyMaximumResourceCapability(fullResource1, scheduler); scheduler.nodes.put(mockNode2.getNodeID(), mockNode2); scheduler.updateMaximumAllocation(mockNode2, true); verifyMaximumResourceCapability(fullResource2, scheduler); scheduler.nodes.remove(mockNode2.getNodeID()); scheduler.updateMaximumAllocation(mockNode2, false); verifyMaximumResourceCapability(fullResource1, scheduler); scheduler.nodes.remove(mockNode1.getNodeID()); scheduler.updateMaximumAllocation(mockNode1, false); verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
/** * Process resource update on a node. */ public void updateNodeResource(RMNode nm, ResourceOption resourceOption) { try { writeLock.lock(); SchedulerNode node = getSchedulerNode(nm.getNodeID()); Resource newResource = resourceOption.getResource(); Resource oldResource = node.getTotalResource(); if (!oldResource.equals(newResource)) { // Notify NodeLabelsManager about this change rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(), newResource); // Log resource change LOG.info("Update resource on node: " + node.getNodeName() + " from: " + oldResource + ", to: " + newResource); nodeTracker.removeNode(nm.getNodeID()); // update resource to node node.updateTotalResource(newResource); nodeTracker.addNode((N) node); } else{ // Log resource change LOG.warn("Update resource on node: " + node.getNodeName() + " with the same resource: " + newResource); } } finally { writeLock.unlock(); } }
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080)); when(mockNode1.getUnallocatedResource()).thenReturn(emptyResource); when(mockNode1.getTotalResource()).thenReturn(fullResource1); when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081)); when(mockNode2.getUnallocatedResource()).thenReturn(emptyResource); when(mockNode2.getTotalResource()).thenReturn(fullResource2); verifyMaximumResourceCapability(fullResource2, scheduler); scheduler.nodeTracker.removeNode(mockNode2.getNodeID()); verifyMaximumResourceCapability(fullResource1, scheduler); scheduler.nodeTracker.removeNode(mockNode1.getNodeID()); verifyMaximumResourceCapability(configuredMaximumResource, scheduler); } finally {
private void checkLaunchedContainerNumOnNode(MockRM rm, NodeId nodeId, int numContainers) { CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler(); SchedulerNode node = cs.getSchedulerNode(nodeId); Assert.assertEquals(numContainers, node.getNumContainers()); }
private void updateMaxResources(SchedulerNode node, boolean add) { Resource totalResource = node.getTotalResource(); ResourceInformation[] totalResources; totalResources = totalResource.getResources(); } else { LOG.warn(node.getNodeName() + " reported in with null resources, which " + "indicates a problem in the source code. Please file an issue at " + "https://issues.apache.org/jira/secure/CreateIssue!default.jspa");
Assert.assertEquals(2, sn1.getNumContainers()); Assert.assertEquals(1 * GB, sn1.getUnallocatedResource().getMemorySize()); .newInstance(Priority.newInstance(0), "*", Resources.createResource(2 * GB), 5)), null); cs.handle(new NodeUpdateSchedulerEvent(sn1.getRMNode())); Assert.assertEquals(1, schedulerApp.getReservedContainers().size()); for (RMContainer rmContainer : sn1.getCopiedListOfRunningContainers()) { if (rmContainer.getContainerId().getContainerId() != 1) { cs.completedContainer(rmContainer, ContainerStatus Assert.assertEquals(7 * GB, sn1.getUnallocatedResource().getMemorySize()); Mockito.any(ResourceCommitRequest.class), Mockito.anyBoolean()); spyCs.handle(new NodeUpdateSchedulerEvent(sn1.getRMNode()));