public MockNM registerNode(String nodeIdStr, int memory, int vCores) throws Exception { MockNM nm = new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService()); nm.registerNode(); return nm; }
public MockNM registerNode(String nodeIdStr, int memory) throws Exception { MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService()); nm.registerNode(); return nm; }
public MockNM registerNode(String nodeIdStr, int memory, int vCores) throws Exception { MockNM nm = new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService()); nm.registerNode(); drainEventsImplicitly(); return nm; }
public MockNM registerNode(String nodeIdStr, int memory) throws Exception { MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService()); nm.registerNode(); drainEventsImplicitly(); return nm; }
public MockNM registerNode(String nodeIdStr, int memory, int vCores, List<ApplicationId> runningApplications) throws Exception { MockNM nm = new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService(), YarnVersionInfo.getVersion()); nm.registerNode(runningApplications); return nm; }
public MockNM registerNode(String nodeIdStr, Resource nodeCapability) throws Exception { MockNM nm = new MockNM(nodeIdStr, nodeCapability, getResourceTrackerService()); nm.registerNode(); drainEventsImplicitly(); return nm; }
public MockNM registerNode(String nodeIdStr, int memory, int vCores, List<ApplicationId> runningApplications) throws Exception { MockNM nm = new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService(), YarnVersionInfo.getVersion()); nm.registerNode(runningApplications); drainEventsImplicitly(); return nm; }
@Test public void testSetRMIdentifierInRegistration() throws Exception { Configuration conf = new Configuration(); rm = new MockRM(conf); rm.start(); MockNM nm = new MockNM("host1:1234", 5120, rm.getResourceTrackerService()); RegisterNodeManagerResponse response = nm.registerNode(); // Verify the RMIdentifier is correctly set in RegisterNodeManagerResponse Assert.assertEquals(ResourceManager.getClusterTimeStamp(), response.getRMIdentifier()); }
protected void setupNodes(MockRM newMockRM) throws Exception { NodeLabel ssdLabel = Records.newRecord(NodeLabel.class); ssdLabel.setName(NODEL_LABEL_SSD); ssdLabel.setExclusivity(true); nm1 = // label = SSD new MockNM("h1:1234", Resource.newInstance(NODE_MEMORY * GB, NODE1_VCORES), newMockRM.getResourceTrackerService(), YarnVersionInfo.getVersion(), new HashSet<NodeLabel>() {{ add(ssdLabel); }}); nm1.registerNode(); NodeLabel gpuLabel = Records.newRecord(NodeLabel.class); ssdLabel.setName(NODEL_LABEL_GPU); ssdLabel.setExclusivity(true); //Label = GPU nm2 = new MockNM("h2:1234", Resource.newInstance(NODE_MEMORY * GB, NODE2_VCORES), newMockRM.getResourceTrackerService(), YarnVersionInfo.getVersion(), new HashSet<NodeLabel>() {{ add(gpuLabel); }}); nm2.registerNode(); nm3 = // label = "" new MockNM("h3:1234", NODE_MEMORY * GB, NODE3_VCORES, newMockRM .getResourceTrackerService ()); nm3.registerNode(); }
@Test public void testSetRMIdentifierInRegistration() throws Exception { Configuration conf = new Configuration(); rm = new MockRM(conf); rm.start(); MockNM nm = new MockNM("host1:1234", 5120, rm.getResourceTrackerService()); RegisterNodeManagerResponse response = nm.registerNode(); // Verify the RMIdentifier is correctly set in RegisterNodeManagerResponse Assert.assertEquals(ResourceManager.getClusterTimeStamp(), response.getRMIdentifier()); }
@Test (timeout = 20000) public void testKillAppWhenFailoverHappensAtRunningState() throws Exception { startRMs(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // failover and kill application // The application is at RUNNING State when failOver happens. // Since RMStateStore has already saved ApplicationState, the active RM // will load the ApplicationState. After that, the application will be at // ACCEPTED State. Because the application is not at Final State, // KillApplicationResponse.getIsKillCompleted is expected to return false. failOverAndKillApp(app0.getApplicationId(), am0.getApplicationAttemptId(), RMAppState.RUNNING, RMAppAttemptState.RUNNING, RMAppState.ACCEPTED); }
@Test (timeout = 20000) public void testKillAppWhenFailoverHappensAtRunningState() throws Exception { startRMs(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // failover and kill application // The application is at RUNNING State when failOver happens. // Since RMStateStore has already saved ApplicationState, the active RM // will load the ApplicationState. After that, the application will be at // ACCEPTED State. Because the application is not at Final State, // KillApplicationResponse.getIsKillCompleted is expected to return false. failOverAndKillApp(app0.getApplicationId(), am0.getApplicationAttemptId(), RMAppState.RUNNING, RMAppAttemptState.RUNNING, RMAppState.ACCEPTED); }
@SuppressWarnings("unchecked") @Test(timeout = 10000) public void testDecommissioningNodeReconnect() throws Exception { MockRM rm = new MockRM(); rm.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm1.registerNode(); rm.waitForState(nm1.getNodeId(), NodeState.RUNNING); rm.getRMContext().getDispatcher().getEventHandler().handle( new RMNodeEvent(nm1.getNodeId(), RMNodeEventType.GRACEFUL_DECOMMISSION)); rm.waitForState(nm1.getNodeId(), NodeState.DECOMMISSIONING); MockNM nm2 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); RegisterNodeManagerResponse response = nm2.registerNode(); // not SHUTDOWN Assert.assertTrue(response.getNodeAction().equals(NodeAction.NORMAL)); rm.stop(); }
@Test (timeout = 60000) public void testAppCleanupWhenNMReconnects() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); // start RM MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED); // wait for application cleanup message received waitForAppCleanupMessageRecved(nm1, app0.getApplicationId()); // reconnect NM with application still active nm1.registerNode(Arrays.asList(app0.getApplicationId())); waitForAppCleanupMessageRecved(nm1, app0.getApplicationId()); rm1.stop(); }
@Test (timeout = 20000) public void testKillAppWhenFailOverHappensDuringApplicationKill() throws Exception { // create a customized ClientRMService // When receives the killApplicationRequest, simply return the response // and make sure the application will not be KILLED State startRMsWithCustomizedClientRMService(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // ensure that the app is in running state Assert.assertEquals(app0.getState(), RMAppState.RUNNING); // kill the app. rm1.killApp(app0.getApplicationId()); // failover happens before this application goes to final state. // The RMAppState that will be loaded by the active rm // should be ACCEPTED. failOverAndKillApp(app0.getApplicationId(), am0.getApplicationAttemptId(), RMAppState.RUNNING, RMAppAttemptState.RUNNING, RMAppState.ACCEPTED); }
/** * Test validateAndCreateResourceRequest fails on recovery, app should ignore * this Exception and continue */ @Test (timeout = 30000) public void testAppFailToValidateResourceRequestOnRecovery() throws Exception{ rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Change the config so that validateAndCreateResourceRequest throws // exception on recovery conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 50); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 100); rm2 = new MockRM(conf, rm1.getRMStateStore()); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); rm2.start(); }
@Test (timeout = 60000) public void testAppCleanupWhenNMReconnects() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); // start RM MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED); // wait for application cleanup message received waitForAppCleanupMessageRecved(nm1, app0.getApplicationId()); // reconnect NM with application still active nm1.registerNode(Arrays.asList(app0.getApplicationId())); waitForAppCleanupMessageRecved(nm1, app0.getApplicationId()); rm1.stop(); }
@Test (timeout = 20000) public void testKillAppWhenFailoverHappensAtFinalState() throws Exception { startRMs(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // kill the app. rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); rm1.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.KILLED); // failover and kill application // The application is at Killed State and RMStateStore has already // saved this applicationState. After failover happens, the current // active RM will load the ApplicationState whose RMAppState is killed. // Because this application is at Final State, // KillApplicationResponse.getIsKillCompleted is expected to return true. failOverAndKillApp(app0.getApplicationId(), am0.getApplicationAttemptId(), RMAppState.KILLED, RMAppAttemptState.KILLED, RMAppState.KILLED); }
@Test (timeout = 20000) public void testKillAppWhenFailoverHappensAtFinalState() throws Exception { startRMs(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // kill the app. rm1.killApp(app0.getApplicationId()); rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); rm1.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.KILLED); // failover and kill application // The application is at Killed State and RMStateStore has already // saved this applicationState. After failover happens, the current // active RM will load the ApplicationState whose RMAppState is killed. // Because this application is at Final State, // KillApplicationResponse.getIsKillCompleted is expected to return true. failOverAndKillApp(app0.getApplicationId(), am0.getApplicationAttemptId(), RMAppState.KILLED, RMAppAttemptState.KILLED, RMAppState.KILLED); }
@Test (timeout = 60000) public void testInvalidatedAMHostPortOnAMRestart() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // a failed app RMApp app2 = rm1.submitApp(200); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1 .nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am2.waitForState(RMAppAttemptState.FAILED); rm1.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED); // before new attempt is launched, the app report returns the invalid AM // host and port. GetApplicationReportRequest request1 = GetApplicationReportRequest.newInstance(app2.getApplicationId()); ApplicationReport report1 = rm1.getClientRMService().getApplicationReport(request1) .getApplicationReport(); Assert.assertEquals("N/A", report1.getHost()); Assert.assertEquals(-1, report1.getRpcPort()); }