public void addRequests(String[] hosts, int memory, int priority, int containers) throws Exception { addRequests(hosts, memory, priority, containers, 0L); }
public AllocateResponse allocate( String host, int memory, int numContainers, List<ContainerId> releases, String labelExpression) throws Exception { return allocate(host, memory, numContainers, 1, releases, labelExpression); }
public List<Container> allocateAndWaitForContainers(int nContainer, int memory, MockNM nm) throws Exception { return allocateAndWaitForContainers("ANY", nContainer, memory, nm); }
public AllocateResponse allocate( String host, int memory, int numContainers, int priority, List<ContainerId> releases, String labelExpression) throws Exception { List<ResourceRequest> reqs = createReq(new String[] { host }, memory, priority, numContainers, labelExpression, -1); return allocate(reqs, releases); }
public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM nm, MockAM am) throws Exception { FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( FinalApplicationStatus.SUCCEEDED, "", ""); am.unregisterAppAttempt(req,true); am.waitForState(RMAppAttemptState.FINISHING); nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am.waitForState(RMAppAttemptState.FINISHED); rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED); }
public AllocateResponse allocate(List<ResourceRequest> resourceRequest, List<SchedulingRequest> newSchedulingRequests, List<ContainerId> releases) throws Exception { final AllocateRequest req = AllocateRequest.newInstance(0, 0F, resourceRequest, releases, null); if (newSchedulingRequests != null) { addSchedulingRequest(newSchedulingRequests); } if (!schedulingRequests.isEmpty()) { req.setSchedulingRequests(schedulingRequests); schedulingRequests.clear(); } return allocate(req); }
RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request LOG.info("Waiting for containers to be created for app 1..."); sleep(1000); alloc1Response = am1.schedule();
am0.registerAppAttempt(); am0.allocate("127.0.0.1", 1000, numContainers, new ArrayList<ContainerId>()); nm1.nodeHeartbeat(true); List<Container> conts = am0.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers(); while (conts.size() < 2) { nm1.nodeHeartbeat(true); conts.addAll(am0.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers()); Thread.sleep(100); (MockMemoryRMStateStore) rm2.getRMStateStore(); rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED); rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED); rm2.getRMContext().getRMApps().get(app0.getApplicationId()); NMContainerStatus container1 = TestRMRestart .createNMContainerStatus(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING); NMContainerStatus container2 = TestRMRestart .createNMContainerStatus(am0.getApplicationAttemptId(), 2, ContainerState.RUNNING); nm1.registerNode(Arrays.asList(container1, container2), null); waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId()); am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
@Test(timeout=1200000) public void testAllocateAfterUnregister() throws Exception { MockRM rm = new MockRM(conf); rm.start(); // Register node1 MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application RMApp app1 = rm.submitApp(2048); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); // unregister app attempt FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( FinalApplicationStatus.KILLED, "", ""); am1.unregisterAppAttempt(req, false); // request container after unregister am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); nm1.nodeHeartbeat(true); rm.drainEvents(); alloc1Response = am1.schedule(); Assert.assertEquals(0, alloc1Response.getAllocatedContainers().size()); }
RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request LOG.info("Waiting for containers to be created for app 1..."); sleep(1000); alloc1Response = am1.schedule(); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId()); am2.registerAppAttempt(); am2.addContainerToBeReleased(cId); try { am2.schedule(); fail("Exception was expected!!"); } catch (InvalidContainerReleaseException e) {
@Test public void testAllocateContainerOnNodeWithoutOffSwitchSpecified() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); RMApp app1 = rm.submitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); // add request for containers List<ResourceRequest> requests = new ArrayList<ResourceRequest>(); requests.add(am1.createResourceReq("127.0.0.1", 1 * GB, 1, 1)); requests.add(am1.createResourceReq("/default-rack", 1 * GB, 1, 1)); am1.allocate(requests, null); // send the request try { // kick the schedule nm1.nodeHeartbeat(true); } catch (NullPointerException e) { Assert.fail("NPE when allocating container on node but " + "forget to set off-switch request should be handled"); } rm.stop(); }
RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); am1.allocate(am1.createReq( new String[] {"127.0.0.1"}, 2 * GB, 1, 1, 10L), null); am1.allocate(am1.createReq( new String[] {"127.0.0.2"}, 2 * GB, 1, 2, 20L), null);
am0.allocateAndWaitForContainers(noOfContainers, containerMemory, nm1); Container container = allocateContainers.get(0); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), container.getId() .getContainerId(), ContainerState.RUNNING); AllocateRequest.newInstance(0, 0F, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), null); AllocateResponse allocate = am0.allocate(req); List<ContainerStatus> completedContainersStatuses = allocate.getCompletedContainersStatuses(); nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE); waitForClusterMemory(nm1, rs, 0);
am0.registerAppAttempt(); am0.allocateAndWaitForContainers(3, CONTAINER_MEMORY, nm1); waitForNewAttemptCreated(app0, am0.getApplicationAttemptId()); am1.registerAppAttempt(); am1.allocateAndWaitForContainers(3, CONTAINER_MEMORY, nm1); for (Container c : allocatedContainers) { cs.markContainerForKillable(schedulerAppAttempt.getRMContainer(c.getId()));
am.allocate(Records.newRecord(AllocateRequest.class)); Assert.assertNull(response.getAMRMToken()); Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps() .get(app.getApplicationId()) .getRMAppAttempt(am.getApplicationAttemptId()).getAMRMToken(); response = am.allocate(Records.newRecord(AllocateRequest.class)); Assert.assertNotNull(response.getAMRMToken()); am.getApplicationAttemptId().toString(), new String[0]); ugi.addTokenIdentifier(oldToken.decodeIdentifier()); response = am.doAllocateAs(ugi, Records.newRecord(AllocateRequest.class)); Assert.assertNotNull(response.getAMRMToken()); verify(spySecretMgr, never()).createAndGetAMRMToken(isA(ApplicationAttemptId.class)); response = am.allocate(Records.newRecord(AllocateRequest.class)); Assert.assertNull(response.getAMRMToken()); response = am.allocate(Records.newRecord(AllocateRequest.class)); Assert.assertNull(response.getAMRMToken()); rm.stop();
am1.registerAppAttempt(); List<Container> allocated1 = am1.allocateAndWaitForContainers("127.0.0.1", NUM_CONTAINERS, 2 * GB, nm1); am2.registerAppAttempt(); am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>()); List<Container> allocated2 = am2.allocateAndWaitForContainers("127.0.0.1", 2, 2 * GB, nm1); am2.allocate("127.0.0.1", 2 * GB, 3, new ArrayList<ContainerId>()); List<Container> allocated3 = am1.allocateAndWaitForContainers("127.0.0.1", 1, 2 * GB, nm1);
resourceRequest.add(am1.createResourceReq("/default-rack", 1024, 3, 1, RMNodeLabelsManager.NO_LABEL)); resourceRequest.add(am1.createResourceReq("*", 1024, 3, 5, "y")); resourceRequest.add(am1.createResourceReq("h1:1234", 1024, 3, 2, RMNodeLabelsManager.NO_LABEL)); resourceRequest.add(am1.createResourceReq("*", 1024, 2, 3, "y")); resourceRequest.add(am1.createResourceReq("h2:1234", 1024, 2, 4, null)); resourceRequest.add(am1.createResourceReq("*", 1024, 4, 3, null)); resourceRequest.add(am1.createResourceReq("h2:1234", 1024, 4, 4, null)); am1.allocate(resourceRequest, new ArrayList<ContainerId>()); CapacityScheduler cs = (CapacityScheduler) rm1.getRMContext().getScheduler(); FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId()); checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 2, "y"); checkNodePartitionOfRequestedPriority(app.getAppSchedulingInfo(), 3, "y"); newReq.add(am1.createResourceReq("h2:1234", 1024, 3, 4, null)); newReq.add(am1.createResourceReq("*", 1024, 3, 5, "z")); newReq.add(am1.createResourceReq("h1:1234", 1024, 3, 4, null)); newReq.add(am1.createResourceReq("*", 1024, 4, 5, "z")); am1.allocate(newReq, new ArrayList<ContainerId>()); resourceRequest1.add(am1.createResourceReq("/default-rack", 1024, 3, 1, null)); resourceRequest1.add(am1.createResourceReq("*", 1024, 3, 5, null)); resourceRequest1.add(am1.createResourceReq("h1:1234", 1024, 3, 2, RMNodeLabelsManager.NO_LABEL)); resourceRequest1.add(am1.createResourceReq("/default-rack", 1024, 2, 1,
containerManager.maxAppAttempts); MockAM am = new MockAM(rm.getRMContext(), rm .getApplicationMasterService(), appAttemptId); am.registerAppAttempt(); am.unregisterAppAttempt(); am.waitForState(RMAppAttemptState.FINISHED); am.waitForState(RMAppAttemptState.FINISHED); rm.stop();
containerManager.maxAppAttempts); MockAM am = new MockAM(rm.getRMContext(), rm .getApplicationMasterService(), appAttemptId); am.registerAppAttempt(); am.unregisterAppAttempt(); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED); rm.stop();
private void waitContainerAllocated(MockAM am, int mem, int nContainer, int startContainerId, MockRM rm, MockNM nm) throws Exception { for (int cId = startContainerId; cId < startContainerId + nContainer; cId++) { am.allocate("*", mem, 1, new ArrayList<ContainerId>()); ContainerId containerId = ContainerId.newContainerId(am.getApplicationAttemptId(), cId); Assert.assertTrue(rm.waitForState(nm, containerId, RMContainerState.ALLOCATED)); } }