0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); allocator.sendFailure(f1); allocator.sendFailure(f2); allocator.sendFailure(f3);
reqMapEvents = ContainerRequestCreator.createRequest(jobId, 0, Resource.newInstance(1024, 1), new String[]{"map"}); allocator.sendRequests(Arrays.asList(reqMapEvents)); Resource.newInstance(2048, 1), new String[] {"reduce"}, false, true); allocator.sendRequests(Arrays.asList(reqReduceEvents)); allocator.schedule(); allocator.schedule(); allocator.close();
private static AppContext createAppContext( ApplicationAttemptId appAttemptId, Job job, Clock clock) { AppContext context = createAppContext(appAttemptId, job); when(context.getClock()).thenReturn(clock); return context; }
0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm1, conf, appAttemptId, mockJob); Resource.newInstance(1024, 1), new String[]{"h1"}); allocator.sendRequest(event1); Resource.newInstance(2048, 1), new String[] {"h1", "h2"}); allocator.sendRequest(event2); allocator.sendFailure(f1); allocator.schedule(); rm1.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assignedContainers = allocator.schedule(); rm1.drainEvents(); Assert.assertEquals("No of assignments must be 2", 2, assignedContainers = allocator.schedule(); Assert.assertEquals("No of assignments must be 0", 0, assignedContainers.size()); Resource.newInstance(1000, 1), new String[]{"h1"}); allocator.sendRequest(event3);
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); Resource.newInstance(1024, 1), new String[] {"h1"}); allocator.sendRequest(event1); List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); assigned = allocator.schedule(); rm.drainEvents(); assertBlacklistAdditionsAndRemovals(0, 0, rm); allocator.sendFailure(f1); Resource.newInstance(1024, 1), new String[] {"h1"}, true, false); allocator.sendRequest(event1f); assigned = allocator.schedule(); rm.drainEvents(); assertBlacklistAdditionsAndRemovals(1, 0, rm); Resource.newInstance(1024, 1), new String[] {"h1", "h3"});
allocator.sendRequests(Arrays.asList(reqMapEvents)); allocator.sendRequests(Arrays.asList(reqReduceEvents)); allocator.schedule(); allocator.schedule(); allocator.schedule(); Assert.assertEquals(2, mockScheduler.lastAnyAskMap); allocator.schedule(); allocator.schedule(); Assert.assertEquals(3, mockScheduler.lastAnyAskMap); ContainerId cid2 = mockScheduler.assignContainer("h2", false); ContainerId cid3 = mockScheduler.assignContainer("h3", false); allocator.schedule(); allocator.schedule(); Assert.assertEquals(0, mockScheduler.lastAnyAskMap); allocator.schedule(); allocator.schedule(); Assert.assertEquals(1, mockScheduler.lastAnyAskMap); allocator.schedule(); allocator.schedule(); Assert.assertEquals(0, mockScheduler.lastAnyAskMap); allocator.schedule(); allocator.schedule();
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); ControlledClock clock = (ControlledClock)allocator.getContext().getClock(); clock.setTime(System.currentTimeMillis()); Resource.newInstance(1024, 1), new String[]{"h1"}); allocator.sendRequest(event1); ContainerRequestEvent event2 = ContainerRequestCreator.createRequest(jobId, 2, Resource.newInstance(1024, 1), new String[]{"h2"}); allocator.sendRequest(event2); ContainerRequestEvent event3 = createRequest(jobId, 3, Resource.newInstance(1024, 1), new String[]{"h2"}, false, true); allocator.sendRequest(event3); allocator.schedule(); rm.drainEvents(); createRequest(jobId, 4, Resource.newInstance(1024, 1), new String[]{"h3"}, false, true); allocator.sendRequest(event4); allocator.schedule();
when(mockTask.getAttempt((TaskAttemptId)any())).thenReturn(mockTaskAttempt); when(mockTaskAttempt.getProgress()).thenReturn(0.01f); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); Resource.newInstance(1024, 1), new String[]{"h1"}); allocator.sendRequest(event1); ContainerRequestEvent event2 = ContainerRequestCreator.createRequest(jobId, 2, Resource.newInstance(1024, 1), new String[]{"h2"}); allocator.sendRequest(event2); ContainerRequestEvent event3 = createRequest(jobId, 3, Resource.newInstance(1024, 1), new String[]{"h1"}, false, true); allocator.sendRequest(event3); allocator.schedule(); rm.drainEvents(); createRequest(jobId, 4, Resource.newInstance(1024, 1), new String[] {"h3"}, false, true); allocator.sendRequest(event4); allocator.schedule(); rm.drainEvents();
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); Resource.newInstance(1024, 1), new String[] {"h1"}); allocator.sendRequest(event1); Resource.newInstance(1024, 1), new String[] {"h2"}); allocator.sendRequest(event2); Resource.newInstance(1024, 1), new String[] {"h3"}); allocator.sendRequest(event3); List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); allocator.sendFailure(f1); ContainerFailedEvent f2 = createFailEvent(jobId, 1, "h2", false); allocator.sendFailure(f2); assigned = allocator.schedule(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); rm.drainEvents();
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); allocator.sendRequest(event1); allocator.sendRequest(event2); List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); allocator.sendRequest(event3); assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); rm.drainEvents(); assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals(0, rm.getMyFifoScheduler().lastAsk.size()); allocator.schedule(); rm.drainEvents(); Assert.assertEquals(5, rm.getMyFifoScheduler().lastAsk.size());
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); jobId, 1, Resource.newInstance(1024, 1), new String[]{"h1"}); allocator.sendRequest(event1); ContainerRequestEvent event2 = ContainerRequestCreator.createRequest( jobId, 2, Resource.newInstance(1024, 1), new String[]{"h1"}); allocator.sendRequest(event2); ContainerRequestEvent event3 = ContainerRequestCreator.createRequest( jobId, 3, Resource.newInstance(1024, 1), new String[]{"h2"}); allocator.sendRequest(event3); List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); rm.drainEvents(); assigned = allocator.schedule(); rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] {event1, event2, event3},
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); ControlledClock clock = (ControlledClock)allocator.getContext().getClock(); clock.setTime(System.currentTimeMillis()); Resource.newInstance(1024, 1), new String[]{"h1"}); allocator.sendRequest(event1); ContainerRequestEvent event2 = ContainerRequestCreator.createRequest(jobId, 2, Resource.newInstance(1024, 1), new String[]{"h2"}); allocator.sendRequest(event2); ContainerRequestEvent event3 = createRequest(jobId, 3, Resource.newInstance(1024, 1), new String[]{"h2"}, false, true); allocator.sendRequest(event3); allocator.schedule(); rm.drainEvents(); createRequest(jobId, 4, Resource.newInstance(1024, 1), new String[] {"h3"}, false, true); allocator.sendRequest(event4);
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob, SystemClock.getInstance()); Resource.newInstance(2048, 1), new String[] {"h1", "h2"}, true, false); allocator.sendRequest(event1); Resource.newInstance(3000, 1), new String[] {"h1"}, false, true); allocator.sendRequest(event2); Resource.newInstance(2048, 1), new String[] {"h3"}, false, false); allocator.sendRequest(event3); List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); rm.drainEvents(); assigned = allocator.schedule(); rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] {event1, event3},
TimelineV2Client client = spy(TimelineV2Client.createTimelineClient(appId)); client.init(conf); when(((RunningAppContext)allocator.getContext()).getTimelineV2Client()). thenReturn(client); allocator.schedule(); verify(client).setTimelineCollectorInfo(collectorInfo); collectorInfo = CollectorInfo.newInstance(localAddr, token); mockScheduler.updateCollectorInfo(collectorInfo); allocator.schedule(); verify(client).setTimelineCollectorInfo(collectorInfo); assertEquals(collectorToken1.decodeIdentifier(), tokens.get(0).decodeIdentifier()); allocator.close();
new String[] {"h" + i}); allocator.sendRequests(Arrays.asList(reqMapEvents)); new String[] {}, false, true); allocator.sendRequests(Arrays.asList(reqReduceEvents)); allocator.schedule(); allocator.schedule(); allocator.schedule(); allocator.close();
ControlledClock clock = new ControlledClock(null); clock.setTime(1); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob, clock); allocator.setMapResourceRequest(BuilderUtils.newResource(1024, 1)); allocator.setReduceResourceRequest(BuilderUtils.newResource(1024, 1)); RMContainerAllocator.AssignedRequests assignedRequests = allocator.getAssignedRequests(); RMContainerAllocator.ScheduledRequests scheduledRequests = allocator.getScheduledRequests(); ContainerRequestEvent event1 = createRequest(jobId, 1, allocator.preemptReducesIfNeeded(); Assert.assertEquals("The reducer is preeempted too soon", 0, assignedRequests.preemptionWaitingReduces.size()); allocator.preemptReducesIfNeeded(); Assert.assertEquals("The reducer is not preeempted", 1, assignedRequests.preemptionWaitingReduces.size());
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); jobId, 1, Resource.newInstance(1024, 1), new String[] {"h1"}); allocator.sendRequest(event1); jobId, 2, Resource.newInstance(1024, 1), new String[] {"h2"}); allocator.sendRequest(event2); List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); rm.drainEvents(); assigned = allocator.schedule(); rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] {event1, event2},
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob); Resource.newInstance(1024, 1), new String[] {"h1"}); allocator.sendRequest(event); TaskAttemptId attemptId = event.getAttemptID(); List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size()); Assert.assertEquals(3, allocator.getJobUpdatedNodeEvents().get(0).getUpdatedNodes().size()); allocator.getJobUpdatedNodeEvents().clear(); assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals(1, assigned.size()); Assert.assertEquals(nm1.getNodeId(), assigned.get(0).getContainer().getNodeId()); Assert.assertTrue(allocator.getJobUpdatedNodeEvents().isEmpty()); Assert.assertTrue(allocator.getTaskAttemptKillEvents().isEmpty()); assigned = allocator.schedule(); rm.drainEvents(); Assert.assertEquals(0, assigned.size()); Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size()); Assert.assertEquals(1, allocator.getTaskAttemptKillEvents().size());
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); final MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob, SystemClock.getInstance()); allocator.sendRequest(createRequest(jobId, 0, Resource.newInstance(1024, 1), locations, false, true)); allocator.scheduleAllReduces(); allocator.makeRemoteRequest(); nm.nodeHeartbeat(true); rm.drainEvents(); allocator.sendRequest(createRequest(jobId, 1, Resource.newInstance(1024, 1), locations, false, false)); assignedContainer += allocator.schedule().size(); nm.nodeHeartbeat(true); rm.drainEvents();
MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); final MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob, SystemClock.getInstance()); allocator.sendRequest(createRequest(jobId, 0, Resource.newInstance(1024, 1), locations, false, true)); for (int i = 0; i < 1;) { rm.drainEvents(); i += allocator.schedule().size(); nm.nodeHeartbeat(true); allocator.sendRequest(createRequest(jobId, 0, Resource.newInstance(1024, 1), locations, true, false)); while (allocator.getTaskAttemptKillEvents().size() == 0) { rm.drainEvents(); allocator.schedule().size(); nm.nodeHeartbeat(true); final String killEventMessage = allocator.getTaskAttemptKillEvents().get(0) .getMessage(); Assert.assertTrue("No reducer rampDown preemption message",