public static List<NMContainerStatus> createNMContainerStatusForApp(MockAM am) { List<NMContainerStatus> list = new ArrayList<NMContainerStatus>(); NMContainerStatus amContainer = createNMContainerStatus(am.getApplicationAttemptId(), 1, ContainerState.RUNNING, 1024); NMContainerStatus completedContainer = createNMContainerStatus(am.getApplicationAttemptId(), 2, ContainerState.COMPLETE, 2048); list.add(amContainer); list.add(completedContainer); return list; }
public static List<NMContainerStatus> createNMContainerStatusForApp(MockAM am) { List<NMContainerStatus> list = new ArrayList<NMContainerStatus>(); NMContainerStatus amContainer = createNMContainerStatus(am.getApplicationAttemptId(), 1, ContainerState.RUNNING, 1024); NMContainerStatus completedContainer = createNMContainerStatus(am.getApplicationAttemptId(), 2, ContainerState.COMPLETE, 2048); list.add(amContainer); list.add(completedContainer); return list; }
public static List<NMContainerStatus> createNMContainerStatusForApp(MockAM am) { List<NMContainerStatus> list = new ArrayList<NMContainerStatus>(); NMContainerStatus amContainer = TestRMRestart.createNMContainerStatus(am.getApplicationAttemptId(), 1, ContainerState.RUNNING); NMContainerStatus runningContainer = TestRMRestart.createNMContainerStatus(am.getApplicationAttemptId(), 2, ContainerState.RUNNING); NMContainerStatus completedContainer = TestRMRestart.createNMContainerStatus(am.getApplicationAttemptId(), 3, ContainerState.COMPLETE); list.add(amContainer); list.add(runningContainer); list.add(completedContainer); return list; }
public static List<NMContainerStatus> createNMContainerStatusForApp(MockAM am) { List<NMContainerStatus> list = new ArrayList<NMContainerStatus>(); NMContainerStatus amContainer = TestRMRestart.createNMContainerStatus(am.getApplicationAttemptId(), 1, ContainerState.RUNNING); NMContainerStatus runningContainer = TestRMRestart.createNMContainerStatus(am.getApplicationAttemptId(), 2, ContainerState.RUNNING); NMContainerStatus completedContainer = TestRMRestart.createNMContainerStatus(am.getApplicationAttemptId(), 3, ContainerState.COMPLETE); list.add(amContainer); list.add(runningContainer); list.add(completedContainer); return list; }
private MockAM launchAndFailAM(RMApp app, MockRM rm, MockNM nm) throws Exception { MockAM am = launchAM(app, rm, nm); nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED); return am; }
public Boolean get() { // release cache is cleaned up and previous running container is not // recovered return scheduler.getApplicationAttempt(am1.getApplicationAttemptId()) .getPendingRelease().isEmpty() && scheduler.getRMContainer(runningContainer) == null; } }, 1000, 20000);
public Boolean get() { // release cache is cleaned up and previous running container is not // recovered return scheduler.getApplicationAttempt(am1.getApplicationAttemptId()) .getPendingRelease().isEmpty() && scheduler.getRMContainer(runningContainer) == null; } }, 1000, 20000);
public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM nm, MockAM am) throws Exception { FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( FinalApplicationStatus.SUCCEEDED, "", ""); am.unregisterAppAttempt(req,true); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHING); nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm.drainEventsImplicitly(); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED); rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED); }
private static void waitForContainerAllocation(Collection<MockNM> nodes, MockAM am, List<Container> allocatedContainers, List<RejectedSchedulingRequest> rejectedRequests, int containerNum) throws Exception { int attemptCount = 10; while (allocatedContainers.size() < containerNum && attemptCount > 0) { for (MockNM node : nodes) { node.nodeHeartbeat(true); } LOG.info("Waiting for containers to be created for " + am.getApplicationAttemptId().getApplicationId() + "..."); sleep(1000); AllocateResponse allocResponse = am.schedule(); allocatedContainers.addAll(allocResponse.getAllocatedContainers()); rejectedRequests.addAll(allocResponse.getRejectedSchedulingRequests()); attemptCount--; } }
private void waitContainerAllocated(MockAM am, int mem, int nContainer, int startContainerId, MockRM rm, MockNM nm) throws Exception { for (int cId = startContainerId; cId < startContainerId + nContainer; cId++) { am.allocate("*", mem, 1, new ArrayList<ContainerId>()); ContainerId containerId = ContainerId.newContainerId(am.getApplicationAttemptId(), cId); Assert.assertTrue(rm.waitForState(nm, containerId, RMContainerState.ALLOCATED)); } }
private void waitContainerAllocated(MockAM am, int mem, int nContainer, int startContainerId, MockRM rm, MockNM nm) throws Exception { for (int cId = startContainerId; cId < startContainerId + nContainer; cId++) { am.allocate("*", mem, 1, new ArrayList<ContainerId>()); ContainerId containerId = ContainerId.newContainerId(am.getApplicationAttemptId(), cId); Assert.assertTrue(rm.waitForState(nm, containerId, RMContainerState.ALLOCATED, 10 * 1000)); } }
private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm, MockAM am, FinishApplicationMasterRequest req) throws Exception { RMState rmState = ((MemoryRMStateStore) rm.getRMContext().getStateStore()).getState(); Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState(); am.unregisterAppAttempt(req,true); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHING); nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED); rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED); // check that app/attempt is saved with the final state ApplicationStateData appState = rmAppState.get(rmApp.getApplicationId()); Assert .assertEquals(RMAppState.FINISHED, appState.getState()); Assert.assertEquals(RMAppAttemptState.FINISHED, appState.getAttempt(am.getApplicationAttemptId()).getState()); }
private LogAggregationContext getLogAggregationContextFromContainerToken( MockRM rm1, MockNM nm1, LogAggregationContext logAggregationContext) throws Exception { RMApp app2 = rm1.submitApp(200, logAggregationContext); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(true); // request a container. am2.allocate("127.0.0.1", 512, 1, new ArrayList<ContainerId>()); ContainerId containerId = ContainerId.newContainerId(am2.getApplicationAttemptId(), 2); rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED); // acquire the container. List<Container> containers = am2.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers(); Assert.assertEquals(containerId, containers.get(0).getId()); // container token is generated. Assert.assertNotNull(containers.get(0).getContainerToken()); ContainerTokenIdentifier token = BuilderUtils.newContainerTokenIdentifier(containers.get(0) .getContainerToken()); return token.getLogAggregationContext(); }
public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM nm, MockAM am) throws Exception { FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( FinalApplicationStatus.SUCCEEDED, "", ""); am.unregisterAppAttempt(req,true); am.waitForState(RMAppAttemptState.FINISHING); nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am.waitForState(RMAppAttemptState.FINISHED); rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED); }
private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm, MockAM am, FinishApplicationMasterRequest req) throws Exception { RMState rmState = ((MemoryRMStateStore) rm.getRMContext().getStateStore()).getState(); Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState(); am.unregisterAppAttempt(req,true); am.waitForState(RMAppAttemptState.FINISHING); nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am.waitForState(RMAppAttemptState.FINISHED); rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED); // check that app/attempt is saved with the final state ApplicationStateData appState = rmAppState.get(rmApp.getApplicationId()); Assert .assertEquals(RMAppState.FINISHED, appState.getState()); Assert.assertEquals(RMAppAttemptState.FINISHED, appState.getAttempt(am.getApplicationAttemptId()).getState()); }
public void sendAMLaunchFailed(ApplicationAttemptId appAttemptId) throws Exception { MockAM am = new MockAM(getRMContext(), masterService, appAttemptId); waitForState(am.getApplicationAttemptId(), RMAppAttemptState.ALLOCATED); getRMContext().getDispatcher().getEventHandler() .handle(new RMAppAttemptEvent(appAttemptId, RMAppAttemptEventType.LAUNCH_FAILED, "Failed")); drainEventsImplicitly(); }
@Test (timeout = 20000) public void testKillAppWhenFailoverHappensAtRunningState() throws Exception { startRMs(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // failover and kill application // The application is at RUNNING State when failOver happens. // Since RMStateStore has already saved ApplicationState, the active RM // will load the ApplicationState. After that, the application will be at // ACCEPTED State. Because the application is not at Final State, // KillApplicationResponse.getIsKillCompleted is expected to return false. failOverAndKillApp(app0.getApplicationId(), am0.getApplicationAttemptId(), RMAppState.RUNNING, RMAppAttemptState.RUNNING, RMAppState.ACCEPTED); }
@Test (timeout = 20000) public void testKillAppWhenFailoverHappensAtRunningState() throws Exception { startRMs(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // failover and kill application // The application is at RUNNING State when failOver happens. // Since RMStateStore has already saved ApplicationState, the active RM // will load the ApplicationState. After that, the application will be at // ACCEPTED State. Because the application is not at Final State, // KillApplicationResponse.getIsKillCompleted is expected to return false. failOverAndKillApp(app0.getApplicationId(), am0.getApplicationAttemptId(), RMAppState.RUNNING, RMAppAttemptState.RUNNING, RMAppState.ACCEPTED); }
@Test (timeout = 30000) public void testAppWithNoContainers() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); RMApp app = rm.submitApp(2000); //kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt = app.getCurrentAppAttempt(); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); am.registerAppAttempt(); am.unregisterAppAttempt(); nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED); rm.stop(); }
@Test (timeout = 60000) public void testAppCleanupWhenNMReconnects() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); // start RM MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED); // wait for application cleanup message received waitForAppCleanupMessageRecved(nm1, app0.getApplicationId()); // reconnect NM with application still active nm1.registerNode(Arrays.asList(app0.getApplicationId())); waitForAppCleanupMessageRecved(nm1, app0.getApplicationId()); rm1.stop(); }