rm2 = new MockRM(conf, rm1.getRMStateStore()); rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService());
(MockMemoryRMStateStore) rm1.getRMStateStore(); MockNM nm = new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService()); RMState state = rm1.getRMStateStore().loadState();
@Test(timeout = 60000) public void testUnManagedRMRestart() throws Exception { // Create RM MockRM rm1 = createMockRM(conf); MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore(); rm1.start(); // create app and launch the AM RMApp app0 = rm1.submitApp(null, "name", "user", new HashMap<>(), true, "default"); rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); // start new RM MockRM rm2 = createMockRM(conf, memStore); rm2.start(); Assert.assertEquals(1, rm2.getRMContext().getRMApps().size()); ApplicationReport appReport = rm2.getClientRMService().getApplicationReport( GetApplicationReportRequest.newInstance(app0.getApplicationId())) .getApplicationReport(); Assert.assertEquals(true, appReport.isUnmanagedApp()); rm1.stop(); rm2.stop(); }
@Test (timeout = 60000) public void testAppSubmissionWithOldDelegationTokenAfterRMRestart() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032"); UserGroupInformation.setConfiguration(conf); MockRM rm1 = new TestSecurityMockRM(conf); rm1.start(); GetDelegationTokenRequest request1 = GetDelegationTokenRequest.newInstance("renewer1"); UserGroupInformation.getCurrentUser().setAuthenticationMethod( AuthMethod.KERBEROS); GetDelegationTokenResponse response1 = rm1.getClientRMService().getDelegationToken(request1); Token<RMDelegationTokenIdentifier> token1 = ConverterUtils.convertFromYarn(response1.getRMDelegationToken(), rmAddr); // start new RM MockRM rm2 = new TestSecurityMockRM(conf, rm1.getRMStateStore()); rm2.start(); // submit an app with the old delegation token got from previous RM. Credentials ts = new Credentials(); ts.addToken(token1.getService(), token1); RMApp app = rm2.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts); rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); }
@Test (timeout = 30000) public void testRMRestartWithRemovedQueue() throws Exception{ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, ""); rm1 = new MockRM(conf); rm1.start(); MockMemoryRMStateStore memStore = (MockMemoryRMStateStore) rm1.getRMStateStore(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); final RMApp app1 = rm1.submitApp(1024, "app1", USER_1, null); MockAM am1 = MockRM.launchAndRegisterAM(app1,rm1, nm1); MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{QUEUE_DOESNT_EXIST}); final String noQueue = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_DOESNT_EXIST; csConf.setCapacity(noQueue, 100); rm2 = new MockRM(csConf, memStore); rm2.start(); UserGroupInformation user2 = UserGroupInformation.createRemoteUser("user2"); ApplicationReport report = user2.doAs(new PrivilegedExceptionAction<ApplicationReport>() { @Override public ApplicationReport run() throws Exception { return rm2.getApplicationReport(app1.getApplicationId()); } }); Assert.assertNotNull(report); }
/** * Test validateAndCreateResourceRequest fails on recovery, app should ignore * this Exception and continue */ @Test (timeout = 30000) public void testAppFailToValidateResourceRequestOnRecovery() throws Exception{ rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Change the config so that validateAndCreateResourceRequest throws // exception on recovery conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 50); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 100); rm2 = new MockRM(conf, rm1.getRMStateStore()); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); rm2.start(); }
MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore(); Map<ApplicationId, ApplicationStateData> rmAppState = memStore.getState().getApplicationState();
rm1.start(); MockMemoryRMStateStore memStore = (MockMemoryRMStateStore) rm1.getRMStateStore(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
(MockMemoryRMStateStore) rm1.getRMStateStore(); Map<ApplicationId, ApplicationStateData> rmAppState = memStore.getState().getApplicationState();
rm.start(); MemoryRMStateStore memStore = (MemoryRMStateStore) rm.getRMStateStore();
rm1.start(); MockMemoryRMStateStore memStore = (MockMemoryRMStateStore) rm1.getRMStateStore(); RMState rmState = memStore.getState(); MockNM nm1 =
rm1.start(); MockMemoryRMStateStore memStore = (MockMemoryRMStateStore) rm1.getRMStateStore(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
(MockMemoryRMStateStore) rm1.getRMStateStore(); Map<ApplicationId, ApplicationStateData> rmAppState = memStore.getState().getApplicationState();
(MockMemoryRMStateStore) rm1.getRMStateStore(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
@Test (timeout = 60000) public void testContainerCleanupWhenRMRestartedAppNotRegistered() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); // start RM MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING); rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING); // start new RM MockRM rm2 = new MockRM(conf, rm1.getRMStateStore()); rm2.start(); // nm1 register to rm2, and do a heartbeat nm1.setResourceTrackerService(rm2.getResourceTrackerService()); nm1.registerNode(Arrays.asList(app0.getApplicationId())); rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED); // Add unknown container for application unknown to scheduler NodeHeartbeatResponse response = nm1.nodeHeartbeat(am0 .getApplicationAttemptId(), 2, ContainerState.RUNNING); waitForContainerCleanup(rm2, nm1, response); rm1.stop(); rm2.stop(); }
assertNotNull("No AppAttempt found", rmAttempt); rm2 = new MockRM(conf, rm1.getRMStateStore()); rm2.start();
@Test (timeout = 20000) public void testRecoverSchedulerAppAndAttemptSynchronously() throws Exception { // start RM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); rm2 = new MockRM(conf, rm1.getRMStateStore()); rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); // scheduler app/attempt is immediately available after RM is re-started. Assert.assertNotNull(rm2.getResourceScheduler().getSchedulerAppInfo( am0.getApplicationAttemptId())); // getTransferredContainers should not throw NPE. rm2.getResourceScheduler() .getTransferredContainers(am0.getApplicationAttemptId()); List<NMContainerStatus> containers = createNMContainerStatusForApp(am0); nm1.registerNode(containers, null); waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId()); }
@Test (timeout = 600000) public void testAppReregisterOnRMWorkPreservingRestart() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); // start RM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); // Issuing registerAppAttempt() before and after RM restart to confirm // registerApplicationMaster() is idempotent. am0.registerAppAttempt(); // start new RM rm2 = new MockRM(conf, rm1.getRMStateStore()); rm2.start(); rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED); rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED); am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); // retry registerApplicationMaster() after RM restart. am0.registerAppAttempt(true); rm2.waitForState(app0.getApplicationId(), RMAppState.RUNNING); rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.RUNNING); }
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM rm2 = new TestSecurityMockRM(conf, rm1.getRMStateStore()) { protected DelegationTokenRenewer createDelegationTokenRenewer() { return new DelegationTokenRenewer() {
@Test (timeout = 30000) public void testAMContainerStatusWithRMRestart() throws Exception { rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1_1 = rm1.submitApp(1024); MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt(); YarnScheduler scheduler = rm1.getResourceScheduler(); Assert.assertTrue(scheduler.getRMContainer( attempt0.getMasterContainer().getId()).isAMContainer()); // Re-start RM rm2 = new MockRM(conf, rm1.getRMStateStore()); rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); List<NMContainerStatus> am1_1Containers = createNMContainerStatusForApp(am1_1); nm1.registerNode(am1_1Containers, null); // Wait for RM to settle down on recovering containers; waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId()); scheduler = rm2.getResourceScheduler(); Assert.assertTrue(scheduler.getRMContainer( attempt0.getMasterContainer().getId()).isAMContainer()); }