@Test (timeout = 30000) public void testRMRestartWithRemovedQueue() throws Exception{ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, ""); rm1 = new MockRM(conf); rm1.start(); MockMemoryRMStateStore memStore = (MockMemoryRMStateStore) rm1.getRMStateStore(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); final RMApp app1 = rm1.submitApp(1024, "app1", USER_1, null); MockAM am1 = MockRM.launchAndRegisterAM(app1,rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{QUEUE_DOESNT_EXIST}); final String noQueue = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_DOESNT_EXIST; csConf.setCapacity(noQueue, 100); rm2 = new MockRM(csConf, memStore); rm2.start(); UserGroupInformation user2 = UserGroupInformation.createRemoteUser("user2"); ApplicationReport report = user2.doAs(new PrivilegedExceptionAction<ApplicationReport>() { @Override public ApplicationReport run() throws Exception { return rm2.getApplicationReport(app1.getApplicationId()); } }); Assert.assertNotNull(report); }
@Test public void testProxyUserConfiguration() throws Exception { MockRM rm = null; try { rm = new MockRM(conf); rm.start(); // wait for web server starting Thread.sleep(10000); UserGroupInformation proxyUser = UserGroupInformation.createProxyUser( BAR_USER.getShortUserName(), FOO_USER); try { ProxyUsers.getDefaultImpersonationProvider().authorize(proxyUser, ipAddress); } catch (AuthorizationException e) { // Exception is not expected Assert.fail(); } } finally { if (rm != null) { rm.stop(); rm.close(); } } }
@Override protected RMAppManager createRMAppManager() { return spy(super.createRMAppManager()); } };
private void syncNodeLost(MockNM nm) throws Exception { rm.sendNodeStarted(nm); rm.waitForState(nm.getNodeId(), NodeState.RUNNING); rm.sendNodeLost(nm); rm.drainEvents(); }
@Before public void setUp() { this.rm = new MockRM(); rm.start(); amService = rm.getApplicationMasterService(); }
rm1 = new MockRM(schedulerConf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200, "autoCreatedQApp", user, null, queueName); Resource amResources = app1.getAMResourceRequests().get(0).getCapability(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); rm1.clearQueueMetrics(app1); rm2 = new MockRM(schedulerConf, rm1.getRMStateStore()); rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); RMApp recoveredApp1 = rm2.getRMContext().getRMApps().get( app1.getApplicationId()); RMAppAttempt loadedAttempt1 = recoveredApp1.getCurrentAppAttempt(); ((RMNodeImpl) rm2.getRMContext().getRMNodes().get(nm1.getNodeId())) .getLaunchedContainers(); assertTrue(launchedContainers.contains(amContainer.getContainerId())); rm2.waitForState(nm1, amContainer.getContainerId(), RMContainerState.RUNNING); rm2.waitForState(nm1, runningContainer.getContainerId(), RMContainerState.RUNNING); rm2.waitForContainerToComplete(loadedAttempt1, completedContainer);
newConf.setMaximumAMResourcePercentPerPartition( CapacitySchedulerConfiguration.ROOT + ".default", "", 0.2f); MockRM rm1 = new MockRM(newConf); rm1.getRMContext().setNodeLabelManager(mgr); rm1.start(); MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); RMApp app1 = rm1.submitApp(1 * GB, "app", "u1", null, "default"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); RMApp app2 = rm1.submitApp(1 * GB, "app", "u2", null, "default"); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); RMApp app3 = rm1.submitApp(1 * GB, "app", "u3", null, "default"); RMApp app4 = rm1.submitApp(1 * GB, "app", "u4", null, "default"); am2.allocate("*", 1 * GB, 50, null); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); Assert.assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps()); Assert.assertEquals(2, lq.getMetrics().getAppsPending()); rm1.close();
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); rm1 = new MockRM(conf); rm1.start(); (MockMemoryRMStateStore) rm1.getRMStateStore(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app0 = rm1.submitApp(200); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); MockRM.finishAMAndVerifyAppState(app0, rm1, nm1, am0); rm2 = new MockRM(conf, memStore); rm2.start(); am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); am0.unregisterAppAttempt(false);
new CapacitySchedulerConfiguration(conf); setupQueueConfiguration(csConf); rm1 = new MockRM(csConf); rm1.start(); MockMemoryRMStateStore memStore = (MockMemoryRMStateStore) rm1.getRMStateStore(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); MockNM nm2 = new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); nm2.registerNode(); RMApp app1_1 = rm1.submitApp(1024, "app1_1", USER_1, null, A); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); RMApp app1_2 = rm1.submitApp(1024, "app1_2", USER_1, null, A); MockAM am1_2 = MockRM.launchAndRegisterAM(app1_2, rm1, nm2); RMApp app2 = rm1.submitApp(1024, "app2", USER_2, null, B); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); assertEquals(rm1.getApplicationReport(app2.getApplicationId()). getYarnApplicationState(), YarnApplicationState.RUNNING); RMApp appNA = rm1.submitApp(1024, "app1_2", USER_1, null, QUEUE_DOESNT_EXIST, false); rm1.clearQueueMetrics(app1_1); rm1.clearQueueMetrics(app1_2); rm1.clearQueueMetrics(app2);
MockRM rm1 = new MockRM(conf, null); rm1.init(conf); rm1.start(); rm1.getAdminService().transitionToStandby(requestInfo); MockRM rm2 = new MockRM(conf); MemoryRMStateStore memStore = (MemoryRMStateStore) rm2.getRMStateStore(); rm2.start(); new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService()); nm.registerNode(); rm2.getAdminService().transitionToActive(requestInfo); RMApp app = rm2.submitApp(200, "test_app", "user", null, "test_queue"); RMAppAttempt attempt0 = app.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am0 = rm2.sendAMLaunched(attempt0.getAppAttemptId()); am0.registerAppAttempt(); assertEquals("root.test_queue", app.getQueue()); ((RMContextImpl)rm1.getRMContext()).setStateStore(memStore); rm1.getAdminService().transitionToActive(requestInfo); rm1.drainEvents(); assertEquals(1, rm1.getRMContext().getRMApps().size()); RMApp recoveredApp = rm1.getRMContext().getRMApps().values().iterator().next(); assertEquals("root.test_queue", recoveredApp.getQueue());
@Test(timeout = 60000) public void testResourceOverCommit() throws Exception { int waitCount; MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); RMApp app1 = rm.submitApp(2048); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); UpdateNodeResourceRequest request = UpdateNodeResourceRequest.newInstance(nodeResourceMap); rm.getAdminService().updateNodeResource(request); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); if (null != report_nm1 && report_nm1.getAvailableResource().getMemorySize() != 0) { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
MockRM rm = setUpMove(); AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm.getResourceScheduler(); RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()) .getCurrentApplicationAttemptId(); rm.waitForState(app.getApplicationId(), RMAppState.KILLED); appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.isEmpty()); assertTrue(appsInA.isEmpty()); rm.stop();
new CapacitySchedulerConfiguration(conf); setupQueueConfiguration(csConf); rm1 = new MockRM(csConf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); MockNM nm2 = new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); nm2.registerNode(); RMApp app1_1 = rm1.submitApp(1024, "app1_1", USER_1, null, A); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); RMApp app1_2 = rm1.submitApp(1024, "app1_2", USER_1, null, A); MockAM am1_2 = MockRM.launchAndRegisterAM(app1_2, rm1, nm2); RMApp app2 = rm1.submitApp(1024, "app2", USER_2, null, B); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); rm1.clearQueueMetrics(app1_1); rm1.clearQueueMetrics(app1_2); rm1.clearQueueMetrics(app2); rm2 = new MockRM(csConf, rm1.getRMStateStore()); rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); nm2.setResourceTrackerService(rm2.getResourceTrackerService()); ((AbstractYarnScheduler) rm2.getResourceScheduler()) .getSchedulerApplications();
@Test (timeout = 80000) public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); MockRM rm1 = new MockRM(conf); rm1.start(); RMApp app1 = rm1.submitApp(200); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); RMApp app2 = rm1.submitApp(200); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED); rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED); RMApp app3 = rm1.submitApp(200); MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); rm1.killApp(app3.getApplicationId()); rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED); rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED); YarnApplicationState.FAILED)); GetApplicationsResponse response1 = rm1.getClientRMService().getApplications(request1); List<ApplicationReport> appList1 = response1.getApplicationList();
conf.set(YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT, "40"); rm = new MockRM(conf); rm.start(); new DecommissioningNodesWatcher(rm.getRMContext()); MockNM nm1 = rm.registerNode("host1:1234", 10240); RMNode node1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId()); NodeId id1 = nm1.getNodeId(); rm.waitForState(id1, NodeState.RUNNING); Assert.assertFalse(watcher.checkReadyToBeDecommissioned(id1)); RMApp app = rm.submitApp(2000); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); rm.sendNodeGracefulDecommission(nm1, YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT); rm.waitForState(id1, NodeState.DECOMMISSIONING); MockRM.finishAMAndVerifyAppState(app, rm, nm1, am); rm.waitForState(app.getApplicationId(), RMAppState.FINISHED); Assert.assertEquals(DecommissioningNodeStatus.READY, watcher.checkDecommissioningStatus(id1));
labels.add(amLabel); labels.add(appLabel); MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore(); rm1.start(); rm1.getAdminService().addToClusterNodeLabels( AddToClusterNodeLabelsRequest.newInstance(labels)); ArrayList resReqs = new ArrayList<>(); resReqs.add(amResourceRequest); RMApp app0 = rm1.submitApp(resReqs, appLabel.getName()); rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); rm2.start(); Assert.assertEquals(1, rm2.getRMContext().getRMApps().size()); ApplicationReport appReport = rm2.getClientRMService().getApplicationReport( GetApplicationReportRequest.newInstance(app0.getApplicationId())) .getApplicationReport(); Assert.assertEquals(appLabel.getName(), appReport.getAppNodeLabelExpression()); rm1.stop(); rm2.stop();
rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("host1:1234", 10240); MockNM nm2 = rm.registerNode("host2:5678", 20480); MockNM nm3 = rm.registerNode("host3:4433", 10240); NodeId id1 = nm1.getNodeId(); NodeId id3 = nm3.getNodeId(); rm.waitForState(id1, NodeState.RUNNING); rm.waitForState(id3, NodeState.RUNNING); RMApp app = rm.submitApp(2000); MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); ApplicationAttemptId aaid = app.getCurrentAppAttempt().getAppAttemptId(); nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING); rm.getNodesListManager().refreshNodes(conf, true); rm.waitForState(id1, NodeState.DECOMMISSIONING); rm.waitForState(id3, NodeState.DECOMMISSIONING); rm.waitForState(id1, NodeState.DECOMMISSIONING); rm.waitForState(id3, NodeState.DECOMMISSIONED); nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING); MockRM.finishAMAndVerifyAppState(app, rm, nm1, am); rm.waitForState(app.getApplicationId(), RMAppState.FINISHED); nodeHeartbeat1 = nm1.nodeHeartbeat(aaid, 2, ContainerState.COMPLETE); Assert.assertEquals(NodeAction.SHUTDOWN, nodeHeartbeat1.getNodeAction()); rm.waitForState(id1, NodeState.DECOMMISSIONED);
UserGroupInformation.setConfiguration(conf); MockRM rm1 = new TestSecurityMockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM rm2 = new TestSecurityMockRM(conf, rm1.getRMStateStore()) { protected DelegationTokenRenewer createDelegationTokenRenewer() { return new DelegationTokenRenewer() { nm1.setResourceTrackerService(rm2.getResourceTrackerService()); rm2.start(); NMContainerStatus containerStatus = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 1, rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); am1.registerAppAttempt(true); rm2.waitForState(app1.getApplicationId(), RMAppState.RUNNING); rm2.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED); rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED);
@Test (timeout = 30000) public void testAppSubmissionWithPreviousToken() throws Exception{ MockRM rm = new TestSecurityMockRM(conf, null); rm.start(); final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm1.registerNode(); rm.submitApp(200, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, false); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING); rm.submitApp(200, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING); MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2); Assert.assertTrue(rm.getRMContext().getDelegationTokenRenewer() .getAllTokens().containsKey(token1)); MockRM.finishAMAndVerifyAppState(app1, rm, nm1, am1);