MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager1 = rm.registerNode("h1:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); rm.drainEvents(); rm.drainEvents(); assertBlacklistAdditionsAndRemovals(0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); rm.drainEvents(); assertBlacklistAdditionsAndRemovals(1, 0, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
forcePreemptThresholdSecs); MyResourceManager rm = new MyResourceManager(conf); rm.start(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(8192, 8)); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents();
conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr); final MyResourceManager rm = new MyResourceManager(conf); rm.start(); AMRMTokenSecretManager secretMgr = rm.getRMContext().getAMRMTokenSecretManager(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); final Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps() .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken(); Assert.assertNotNull("app should have a token", oldToken);
MyResourceManager rm1 = new MyResourceManager(conf); rm1.start(); RMApp app = rm1.submitApp(1024); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); nm1.nodeHeartbeat(true); // Node heartbeat rm1.drainEvents(); rm1.sendAMLaunched(appAttemptId); rm1.drainEvents(); rm1.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assignedContainers.size()); rm1.drainEvents(); rm1.drainEvents(); Assert.assertEquals("No of assignments must be 2", 2, assignedContainers.size()); MyResourceManager rm2 = new MyResourceManager(conf, rm1.getRMStateStore()); rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); allocator.updateSchedulerProxy(rm2);
LOG.info("Running testAvoidAskMoreReducersWhenReducerPreemptionIsRequired"); Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager = rm.registerNode("h1:1234", 1024); rm.drainEvents(); rm.drainEvents(); rm.drainEvents(); rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1024, 1)); allocator.schedule(); rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(2048, 0));
MRJobConfig.MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT, 33); MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); nodeManagers[nmNum] = registerNodeManager(nmNum++, rm); nodeManagers[0].nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents();
Configuration conf = new Configuration(); conf.setInt(MRJobConfig.MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC, -1); MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager = rm.registerNode("h1:1234", 4096); rm.drainEvents(); MockNM nodeManager2 = rm.registerNode("h2:1234", 1024); rm.drainEvents(); rm.drainEvents(); rm.drainEvents(); rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(3072, 3)); allocator.schedule();
LOG.info("Running testUpdateAskOnRampDownAllReduces"); Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager = rm.registerNode("h1:1234", 1024); rm.drainEvents(); rm.drainEvents(); rm.drainEvents(); rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1024, 1)); allocator.schedule(); rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(0, 0));
MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager1 = rm.registerNode("h1:1234", 10240); MockNM nodeManager2 = rm.registerNode("h2:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); assertBlacklistAdditionsAndRemovals(2, 0, rm); rm.drainEvents();
final MyResourceManager rm = new MyResourceManager(conf); rm.start(); DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext() .getDispatcher(); RMApp rmApp = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 21504); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); rm.drainEvents(); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); allocator.schedule(); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals(0.05f, job.getProgress(), 0.001f); Assert.assertEquals(0.05f, rmApp.getProgress(), 0.001f); finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1); allocator.schedule(); rm.drainEvents(); Assert.assertEquals(0.095f, job.getProgress(), 0.001f);
MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager1 = rm.registerNode("h1:1234", 10240); MockNM nodeManager2 = rm.registerNode("h2:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); Assert.assertEquals(4, rm.getMyFifoScheduler().lastAsk.size()); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); Assert.assertEquals(3, rm.getMyFifoScheduler().lastAsk.size()); nodeManager2.nodeHeartbeat(true); // Node heartbeat nodeManager3.nodeHeartbeat(true); // Node heartbeat rm.drainEvents();
MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager1 = rm.registerNode("h1:1234", 3072); // can assign 2 maps rm.registerNode("h2:1234", 10240); // wont heartbeat on node local node MockNM nodeManager3 = rm.registerNode("h3:1234", 1536); // assign 1 map rm.drainEvents(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); rm.drainEvents(); rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] {event1, event2, event3}, assigned, false);
@Test public void testUpdatedNodes() throws Exception { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nm1 = rm.registerNode("h1:1234", 10240); MockNM nm2 = rm.registerNode("h2:1234", 10240); rm.drainEvents(); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size()); Assert.assertEquals(3, allocator.getJobUpdatedNodeEvents().get(0).getUpdatedNodes().size()); rm.drainEvents(); Assert.assertEquals(1, assigned.size()); Assert.assertEquals(nm1.getNodeId(), assigned.get(0).getContainer().getNodeId()); rm.drainEvents();
final MyResourceManager rm = new MyResourceManager(conf); rm.start(); DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext() .getDispatcher(); RMApp rmApp = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 11264); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); rm.drainEvents(); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); allocator.schedule(); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals(0.05f, job.getProgress(), 0.001f); Assert.assertEquals(0.05f, rmApp.getProgress(), 0.001f); rm.drainEvents(); Assert.assertEquals(0.14f, job.getProgress(), 0.001f); Assert.assertEquals(0.14f, rmApp.getProgress(), 0.001f);
MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager1 = rm.registerNode("h1:1234", 1024); MockNM nodeManager2 = rm.registerNode("h2:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); nodeManager2.nodeHeartbeat(true); // Node heartbeat nodeManager3.nodeHeartbeat(true); // Node heartbeat rm.drainEvents(); rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] {event1, event3}, assigned, false);
MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); MockNM nodeManager1 = rm.registerNode("h1:1234", 10240); MockNM nodeManager2 = rm.registerNode("h2:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); rm.drainEvents(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); nodeManager2.nodeHeartbeat(true); // Node heartbeat nodeManager3.nodeHeartbeat(true); // Node heartbeat rm.drainEvents(); rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] {event1, event2}, assigned, false);
preemptThreshold); MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents();
final MyResourceManager rm = new MyResourceManager(conf); rm.start(); final RMApp app = rm.submitApp(1024); rm.drainEvents(); final MockNM nm = rm.registerNode(String.format("%s:1234", host), 2048); nm.nodeHeartbeat(true); rm.drainEvents(); final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); rm.drainEvents(); final JobId jobId = MRBuilderUtils .newJobId(appAttemptId.getApplicationId(), 0); rm.drainEvents(); locations, false, true)); for (int i = 0; i < 1;) { rm.drainEvents(); i += allocator.schedule().size(); nm.nodeHeartbeat(true); locations, true, false)); while (allocator.getTaskAttemptKillEvents().size() == 0) { rm.drainEvents(); allocator.schedule().size(); nm.nodeHeartbeat(true);
MyResourceManager rm = new MyResourceManager(conf); rm.start(); RMApp app = rm.submitApp(1024); rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); rm.drainEvents(); rm.sendAMLaunched(appAttemptId); rm.drainEvents();
conf.setInt( MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, 0); MyResourceManager rm1 = new MyResourceManager(conf); rm1.start(); RMApp app = rm1.submitApp(1024); rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); nm1.nodeHeartbeat(true); rm1.drainEvents(); rm1.sendAMLaunched(appAttemptId); rm1.drainEvents(); Assert.assertTrue(e.getMessage().contains("Could not contact RM after")); rm1.drainEvents(); Assert.assertEquals("Should Have 1 Job Event", 1, allocator.jobEvents.size());