private static YarnScheduler mockYarnScheduler() { YarnScheduler yarnScheduler = mock(YarnScheduler.class); when(yarnScheduler.getMinimumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); when(yarnScheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn( Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102))); when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( Arrays.asList(getApplicationAttemptId(103))); ApplicationAttemptId attemptId = getApplicationAttemptId(1); when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null); ResourceCalculator rc = new DefaultResourceCalculator(); when(yarnScheduler.getResourceCalculator()).thenReturn(rc); return yarnScheduler; }
@Override public void handle(SchedulerEvent event) { scheduler.handle(event); } }
/** * Utility method to validate a list resource requests, by insuring that the * requested memory/vcore is non-negative and not greater than max */ public static void normalizeAndValidateRequests(List<ResourceRequest> ask, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException { QueueInfo queueInfo = null; try { queueInfo = scheduler.getQueueInfo(queueName, false, false); } catch (IOException e) { } for (ResourceRequest resReq : ask) { SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, queueName, scheduler, rmContext, queueInfo); } }
/** * Move all apps in the set of queues to the parent plan queue's default * reservation queue in a synchronous fashion */ private void moveAppsInQueueSync(String expiredReservation, String defReservationQueue) { List<ApplicationAttemptId> activeApps = scheduler.getAppsInQueue(expiredReservation); if (activeApps.isEmpty()) { return; } for (ApplicationAttemptId app : activeApps) { // fallback to parent's default queue try { scheduler.moveApplication(app.getApplicationId(), defReservationQueue); } catch (YarnException e) { LOG.warn( "Encountered unexpected error during migration of application: {}" + " from reservation: {}", app, expiredReservation, e); } } }
/** * First sets entitlement of queues to zero to prevent new app submission. * Then move all apps in the set of queues to the parent plan queue's default * reservation queue if move is enabled. Finally cleanups the queue by killing * any apps (if move is disabled or move failed) and removing the queue */ protected void cleanupExpiredQueues(String planQueueName, boolean shouldMove, Set<String> toRemove, String defReservationQueue) { for (String expiredReservationId : toRemove) { try { // reduce entitlement to 0 String expiredReservation = getReservationQueueName(planQueueName, expiredReservationId); setQueueEntitlement(planQueueName, expiredReservation, 0.0f, 0.0f); if (shouldMove) { moveAppsInQueueSync(expiredReservation, defReservationQueue); } if (scheduler.getAppsInQueue(expiredReservation).size() > 0) { scheduler.killAllAppsInQueue(expiredReservation); LOG.info("Killing applications in queue: {}", expiredReservation); } else { scheduler.removeQueue(expiredReservation); LOG.info("Queue: " + expiredReservation + " removed"); } } catch (YarnException e) { LOG.warn("Exception while trying to expire reservation: {}", expiredReservationId, e); } } }
scheduler.getMaximumResourceCapability(), submissionContext.getQueue(), scheduler, isRecovery, rmContext); } catch (InvalidResourceRequestException e) { SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(), scheduler.getClusterResource(), scheduler.getMinimumResourceCapability(), scheduler.getMaximumResourceCapability(), scheduler.getMinimumResourceCapability()); return amReq;
appAttempt.scheduler.allocate(appAttempt.applicationAttemptId, EMPTY_CONTAINER_REQUEST_LIST, EMPTY_CONTAINER_RELEASE_LIST, null, null); .get(0)); RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler .getRMContainer(appAttempt.getMasterContainer().getId()); rmMasterContainer.setAMContainer(true);
rScheduler.getMaximumResourceCapability(), app.getQueue(), rScheduler, rmContext); } catch (InvalidResourceRequestException e) { this.rScheduler.allocate(appAttemptId, ask, release, blacklistAdditions, blacklistRemovals); for(RMNode rmNode: updatedNodes) { SchedulerNodeReport schedulerNodeReport = rScheduler.getNodeReport(rmNode.getNodeID()); Resource used = BuilderUtils.newResource(0, 0); int numContainers = 0; allocateResponse.setAvailableResources(allocation.getResourceLimit()); allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
Assert.assertEquals(0, scheduler.getNumClusterNodes()); int maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores(); Assert.assertEquals(expectedMaxVCores[0], maxVCores); scheduler.handle(new NodeAddedSchedulerEvent(node1)); Assert.assertEquals(1, scheduler.getNumClusterNodes()); maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores(); Assert.assertEquals(expectedMaxVCores[1], maxVCores); scheduler.handle(new NodeRemovedSchedulerEvent(node1)); Assert.assertEquals(0, scheduler.getNumClusterNodes()); maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores(); Assert.assertEquals(expectedMaxVCores[2], maxVCores); scheduler.handle(new NodeAddedSchedulerEvent(node2)); Assert.assertEquals(1, scheduler.getNumClusterNodes()); maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores(); Assert.assertEquals(expectedMaxVCores[3], maxVCores); scheduler.handle(new NodeAddedSchedulerEvent(node3)); Assert.assertEquals(2, scheduler.getNumClusterNodes()); maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores(); Assert.assertEquals(expectedMaxVCores[4], maxVCores); scheduler.handle(new NodeRemovedSchedulerEvent(node3)); Assert.assertEquals(1, scheduler.getNumClusterNodes()); maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores(); Assert.assertEquals(expectedMaxVCores[5], maxVCores);
try { QueueInfo queueInfo = scheduler.getQueueInfo(request.getQueueName(), request.getIncludeChildQueues(), request.getRecursive()); if (request.getIncludeApplications()) { List<ApplicationAttemptId> apps = scheduler.getAppsInQueue(request.getQueueName()); appReports = new ArrayList<ApplicationReport>(apps.size()); for (ApplicationAttemptId app : apps) {
getScheduler().getMaximumResourceCapability(app.getQueue()); } else { try { allocation = getScheduler().allocate(appAttemptId, ask, request.getSchedulingRequests(), release, blacklistAdditions, blacklistRemovals, containerUpdateRequests); .getApplicationAttempt(appAttemptId).pullUpdateContainerErrors()); response.setNumClusterNodes(getScheduler().getNumClusterNodes());
/** * {@link RMAppAttemptState#ALLOCATED} */ @SuppressWarnings("unchecked") private void testAppAttemptAllocatedState(Container amContainer) { assertEquals(RMAppAttemptState.ALLOCATED, applicationAttempt.getAppAttemptState()); assertEquals(amContainer, applicationAttempt.getMasterContainer()); // Check events verify(applicationMasterLauncher).handle(any(AMLauncherEvent.class)); verify(scheduler, times(2)).allocate(any(ApplicationAttemptId.class), any(List.class), any(List.class), any(List.class), any(List.class), any(List.class), any(ContainerUpdates.class)); verify(nmTokenManager).clearNodeSetForAttempt( applicationAttempt.getAppAttemptId()); }
DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); opCtx.updateAllocationParams( getScheduler().getMinimumResourceCapability(), getScheduler().getMaximumResourceCapability(), getScheduler().getMinimumResourceCapability(), tokenExpiryInterval); appAttempt.setOpportunisticContainerContext(opCtx);
.newRecordInstance(RegisterApplicationMasterResponse.class); response.setMaximumResourceCapability(rScheduler .getMaximumResourceCapability(app.getQueue())); response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId) .getSubmissionContext().getAMContainerSpec().getApplicationACLs()); .getSchedulingResourceTypes());
List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); scheduler.killAllAppsInQueue("DOES_NOT_EXIST"); Assert.fail(); } catch (YarnException e) { appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size());
List<NMContainerStatus> containerReports = new ArrayList<>(); containerReports.add(containerReport); scheduler.handle(new NodeAddedSchedulerEvent(node1, containerReports)); RMContainer rmContainer = scheduler.getRMContainer(containerId);
queueInfo = appAttempt.scheduler.getQueueInfo(queue, false, false); } catch (IOException e) { appAttempt.scheduler.allocate( appAttempt.applicationAttemptId, appAttempt.amReqs, null, EMPTY_CONTAINER_RELEASE_LIST,
applicationAttemptId); response.setMaximumResourceCapability(getScheduler() .getMaximumResourceCapability(app.getQueue())); response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId) .getSubmissionContext().getAMContainerSpec().getApplicationACLs()); .getKeepContainersAcrossApplicationAttempts()) { List<Container> transferredContainers = getScheduler() .getTransferredContainers(applicationAttemptId); if (!transferredContainers.isEmpty()) { response.setContainersFromPreviousAttempts(transferredContainers); .getSchedulingResourceTypes()); response.setResourceTypes(ResourceUtils.getResourcesTypeInfo()); if (getRmContext().getYarnConfiguration().getBoolean(
@Override public GetNewApplicationResponse getNewApplication( GetNewApplicationRequest request) throws YarnException { GetNewApplicationResponse response = recordFactory .newRecordInstance(GetNewApplicationResponse.class); response.setApplicationId(getNewApplicationId()); // Pick up min/max resource from scheduler... response.setMaximumResourceCapability(scheduler .getMaximumResourceCapability()); return response; }
RMContainer container = scheduler.getRMContainer(containerId); if (container != null) { this.rmContext.getDispatcher().getEventHandler().handle(