private void sendEnoughNodeUpdatesToAssignFully() { for (RMNode node : rmNodes) { NodeUpdateSchedulerEvent nodeUpdateSchedulerEvent = new NodeUpdateSchedulerEvent(node); for (int i = 0; i < NODE_CAPACITY_MULTIPLE; i++) { scheduler.handle(nodeUpdateSchedulerEvent); } } }
private void sendEnoughNodeUpdatesToAssignFully() { for (RMNode node : rmNodes) { NodeUpdateSchedulerEvent nodeUpdateSchedulerEvent = new NodeUpdateSchedulerEvent(node); for (int i = 0; i < NODE_CAPACITY_MULTIPLE; i++) { scheduler.handle(nodeUpdateSchedulerEvent); } } } }
private void doNMHeartbeat(MockRM rm, NodeId nodeId, int nHeartbeat) { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nodeId); for (int i = 0; i < nHeartbeat; i++) { cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); } }
private void nodeUpdate(NodeManager nm) { RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); resourceManager.getResourceScheduler().handle(nodeUpdate); }
private void nodeUpdate(ResourceManager rm, NodeManager nm) { RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); rm.getResourceScheduler().handle(nodeUpdate); }
private void nodeUpdate( org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm) { RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); resourceManager.getResourceScheduler().handle(nodeUpdate); }
private void nodeUpdate(NodeManager nm) { RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); resourceManager.getResourceScheduler().handle(nodeUpdate); }
private void nodeUpdate( org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1) { RMNode node = resourceManager.getRMContext().getRMNodes().get(nm1.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); resourceManager.getResourceScheduler().handle(nodeUpdate); }
private void nodeUpdate( org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1) { RMNode node = resourceManager.getRMContext().getRMNodes().get(nm1.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); resourceManager.getResourceScheduler().handle(nodeUpdate); }
@Test public void testRemovedNodeDecomissioningNode() throws Exception { // Register nodemanager NodeManager nm = registerNode("host_decom", 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4)); RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); resourceManager.getResourceScheduler().handle(nodeUpdate); // Force remove the node to simulate race condition ((FairScheduler) resourceManager.getResourceScheduler()) .getNodeTracker().removeNode(nm.getNodeId()); // Kick off another heartbeat with the node state mocked to decommissioning RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes() .get(nm.getNodeId())); when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); resourceManager.getResourceScheduler().handle( new NodeUpdateSchedulerEvent(spyNode)); }
@Test public void testRemovedNodeDecomissioningNode() throws Exception { // Register nodemanager NodeManager nm = registerNode("host_decom", 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4)); RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); resourceManager.getResourceScheduler().handle(nodeUpdate); // Force remove the node to simulate race condition ((FifoScheduler) resourceManager.getResourceScheduler()) .getNodeTracker().removeNode(nm.getNodeId()); // Kick off another heartbeat with the node state mocked to decommissioning RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes() .get(nm.getNodeId())); when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); resourceManager.getResourceScheduler().handle( new NodeUpdateSchedulerEvent(spyNode)); }
@Test public void testRemovedNodeDecomissioningNode() throws Exception { // Register nodemanager NodeManager nm = registerNode("host_decom", 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4)); RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()); // Send a heartbeat to kick the tires on the Scheduler NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node); resourceManager.getResourceScheduler().handle(nodeUpdate); // force remove the node to simulate race condition ((CapacityScheduler) resourceManager.getResourceScheduler()).getNodeTracker(). removeNode(nm.getNodeId()); // Kick off another heartbeat with the node state mocked to decommissioning RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes() .get(nm.getNodeId())); when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); resourceManager.getResourceScheduler().handle( new NodeUpdateSchedulerEvent(spyNode)); }
@Test (timeout = 60000) public void testBasic() throws InterruptedException { // Add one node String host = "127.0.0.1"; RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(4096, 4), 1, host); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdateEvent); ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); createMockRMApp(appAttemptId); scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false); scheduler.addApplicationAttempt(appAttemptId, false, false); List<ResourceRequest> ask = new ArrayList<>(); ask.add(createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true)); scheduler.allocate( appAttemptId, ask, null, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS); FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId); triggerSchedulingAttempt(); checkAppConsumption(app, Resources.createResource(1024, 1)); }
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
private void registerNodeAndSubmitApp( int memory, int vcores, int appContainers, int appMemory) { RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(memory, vcores), 1, "node1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); assertEquals("Incorrect amount of resources in the cluster", memory, scheduler.rootMetrics.getAvailableMB()); assertEquals("Incorrect amount of resources in the cluster", vcores, scheduler.rootMetrics.getAvailableVirtualCores()); createSchedulingRequest(appMemory, "queueA", "user1", appContainers); scheduler.update(); // Sufficient node check-ins to fully schedule containers for (int i = 0; i < 3; i++) { NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdate1); } assertEquals("app1's request is not met", memory - appContainers * appMemory, scheduler.rootMetrics.getAvailableMB()); }
@Test (expected = YarnException.class) public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); queueMgr.getLeafQueue("queue2", true); scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2", Resource.newInstance(1024, 1)); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); scheduler.handle(updateEvent); scheduler.handle(updateEvent); assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage()); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); }
@Test public void testNodeUpdateBeforeAppAttemptInit() throws Exception { FifoScheduler scheduler = new FifoScheduler(); MockRM rm = new MockRM(conf); scheduler.setRMContext(rm.getRMContext()); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, rm.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); scheduler.handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = ApplicationId.newInstance(0, 1); scheduler.addApplication(appId, "queue1", "user1", false); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); try { scheduler.handle(updateEvent); } catch (NullPointerException e) { Assert.fail(); } ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1); scheduler.addApplicationAttempt(attId, false, false); rm.stop(); }
@Test (expected = YarnException.class) public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSQueue queue2 = queueMgr.getLeafQueue("queue2", true); queue2.setMaxShare( new ConfigurableResource(Resource.newInstance(1024, 1))); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); scheduler.handle(updateEvent); scheduler.handle(updateEvent); assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage()); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); }
@Test (timeout = 5000) public void testMultipleContainersWaitingForReservation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Request full capacity of node createSchedulingRequest(1024, "queue1", "user1", 1); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue2", "user2", 1); ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue3", "user3", 1); scheduler.update(); scheduler.handle(updateEvent); // One container should get reservation and the other should get nothing assertEquals(1024, scheduler.getSchedulerApp(attId1).getCurrentReservation().getMemory()); assertEquals(0, scheduler.getSchedulerApp(attId2).getCurrentReservation().getMemory()); }