@Override public SchedulerAppReport getSchedulerAppInfo( ApplicationAttemptId appAttemptId) { SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId); if (attempt == null) { if (LOG.isDebugEnabled()) { LOG.debug("Request for appInfo of unknown attempt " + appAttemptId); } return null; } return new SchedulerAppReport(attempt); }
@Override public RMContainer getRMContainer(ContainerId containerId) { SchedulerApplicationAttempt attempt = getCurrentAttemptForContainer(containerId); return (attempt == null) ? null : attempt.getRMContainer(containerId); }
@Override @LimitedPrivate("yarn") @Unstable public Resource getClusterResource() { return super.getClusterResource(); }
@SuppressWarnings("unchecked") @Override public void serviceInit(Configuration conf) throws Exception { ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) scheduler).init(conf); super.serviceInit(conf); }
@SuppressWarnings("unchecked") @Override public void serviceStart() throws Exception { ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) scheduler).start(); super.serviceStart(); }
/** * Normalize a list of resource requests * using queue maximum resource allocations. * @param asks resource requests */ protected void normalizeResourceRequests(List<ResourceRequest> asks, String queueName) { Resource maxAllocation = getMaximumResourceCapability(queueName); for (ResourceRequest ask : asks) { ask.setCapability( getNormalizedResource(ask.getCapability(), maxAllocation)); } }
List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); String queue = scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() .getQueueName(); Assert.assertTrue(queue.equals("a1")); List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1"); assertTrue(appsInB1.isEmpty()); List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b"); assertTrue(appsInB.isEmpty()); scheduler.moveApplication(app.getApplicationId(), "b1"); appsInB1 = scheduler.getAppsInQueue("b1"); assertEquals(1, appsInB1.size()); queue = scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() .getQueueName(); Assert.assertTrue(queue.equals("b1"));
SchedulerNode schedulerNode1 = scheduler.getSchedulerNode(nm1.getNodeId()); .getSchedulerApplications(); SchedulerApplication<SchedulerApplicationAttempt> schedulerApp = sa.get( recoveredApp1.getApplicationId()); schedulerApp.getCurrentAppAttempt(); assertTrue(schedulerAttempt.getLiveContainers() .contains(scheduler.getRMContainer(amContainer.getContainerId()))); assertTrue(schedulerAttempt.getLiveContainers() .contains(scheduler.getRMContainer(runningContainer.getContainerId()))); assertEquals(schedulerAttempt.getCurrentConsumption(), usedResources);
List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); String queue = scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() .getQueueName(); Assert.assertTrue(queue.equals("a1")); List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); scheduler.killAllAppsInQueue("a1"); appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.isEmpty()); appsInA1 = scheduler.getAppsInQueue("a1"); assertTrue(appsInA1.isEmpty()); appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.isEmpty());
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); attempt.getPendingRelease().add(containerId); } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId, null); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
completedContainerInternal(rmContainer, containerStatus, event); completeOustandingUpdatesWhichAreReserved( rmContainer, containerStatus, event); } else { new RMContainerFinishedEvent(containerId, containerStatus, event)); SchedulerApplicationAttempt schedulerAttempt = getCurrentAttemptForContainer(containerId); if (schedulerAttempt != null) { schedulerAttempt.removeRMContainer(containerId); " in state: " + rmContainer.getState() + " event:" + event); getSchedulerNode(rmContainer.getNodeId()).releaseContainer( rmContainer.getContainerId(), false); recoverResourceRequestForContainer(rmContainer);
public Boolean get() { // release cache is cleaned up and previous running container is not // recovered return scheduler.getApplicationAttempt(am1.getApplicationAttemptId()) .getPendingRelease().isEmpty() && scheduler.getRMContainer(runningContainer) == null; } }, 1000, 20000);
@Override public void moveAllApps(String sourceQueue, String destQueue) throws YarnException { try { writeLock.lock(); // check if destination queue is a valid leaf queue try { getQueueInfo(destQueue, false, false); } catch (IOException e) { LOG.warn(e); throw new YarnException(e); } // check if source queue is a valid List<ApplicationAttemptId> apps = getAppsInQueue(sourceQueue); if (apps == null) { String errMsg = "The specified Queue: " + sourceQueue + " doesn't exist"; LOG.warn(errMsg); throw new YarnException(errMsg); } // generate move events for each pending/running app for (ApplicationAttemptId appAttemptId : apps) { this.rmContext.getDispatcher().getEventHandler() .handle(new RMAppManagerEvent(appAttemptId.getApplicationId(), destQueue, RMAppManagerEventType.APP_MOVE)); } } finally { writeLock.unlock(); } }
List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); String queue = scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() .getQueueName(); Assert.assertEquals("a1", queue); List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1"); assertTrue(appsInB1.isEmpty()); List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b"); assertTrue(appsInB.isEmpty()); scheduler.moveAllApps("a1", "b1"); appsInB1 = scheduler.getAppsInQueue("b1"); assertEquals(1, appsInB1.size()); queue = scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() .getQueueName(); Assert.assertEquals("b1", queue);
/** * Rollback container update after expiry. * @param containerId ContainerId. */ protected void rollbackContainerUpdate( ContainerId containerId) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { LOG.info("Cannot rollback resource for container " + containerId + ". The container does not exist."); return; } T app = getCurrentAttemptForContainer(containerId); if (getCurrentAttemptForContainer(containerId) == null) { LOG.info("Cannot rollback resource for container " + containerId + ". The application that the container " + "belongs to does not exist."); return; } if (Resources.fitsIn(rmContainer.getLastConfirmedResource(), rmContainer.getContainer().getResource())) { LOG.info("Roll back resource for container " + containerId); handleDecreaseRequests(app, Arrays.asList( UpdateContainerRequest.newInstance( rmContainer.getContainer().getVersion(), rmContainer.getContainerId(), ContainerUpdateType.DECREASE_RESOURCE, rmContainer.getLastConfirmedResource(), null))); } }
nlm.start(); context.setNodeLabelManager(nlm); scheduler.setRMContext(context); scheduler.init(yarnConf); scheduler.start(); dispatcher.register(SchedulerEventType.class, scheduler); Assert.assertNotNull(context.getRMNodes().get(nodeId1)); Assert.assertTrue(scheduler.getSchedulerNode(nodeId1).getRMNode() == context.getRMNodes().get(nodeId1)); Assert.assertEquals(context.getRMNodes().get(nodeId1). Assert.assertTrue(scheduler.getSchedulerNode(nodeId1).getRMNode() == context.getRMNodes().get(nodeId1)); getTotalCapability(), capability1); nlm.stop(); scheduler.stop();
@Override public void serviceInit(Configuration conf) throws Exception { initScheduler(conf); super.serviceInit(conf); }
@Override public void serviceStart() throws Exception { super.serviceStart(); }
/** * Process resource update on a node. */ public synchronized void updateNodeResource(RMNode nm, ResourceOption resourceOption) { SchedulerNode node = getSchedulerNode(nm.getNodeID()); Resource newResource = resourceOption.getResource(); Resource oldResource = node.getTotalResource(); if(!oldResource.equals(newResource)) { // Log resource change LOG.info("Update resource on node: " + node.getNodeName() + " from: " + oldResource + ", to: " + newResource); nodes.remove(nm.getNodeID()); updateMaximumAllocation(node, false); // update resource to node node.setTotalResource(newResource); nodes.put(nm.getNodeID(), (N)node); updateMaximumAllocation(node, true); // update resource to clusterResource Resources.subtractFrom(clusterResource, oldResource); Resources.addTo(clusterResource, newResource); } else { // Log resource change LOG.warn("Update resource on node: " + node.getNodeName() + " with the same resource: " + newResource); } }
private void dispatchRequestsForPlacement(ApplicationAttemptId appAttemptId, List<SchedulingRequest> schedulingRequests) { if (schedulingRequests != null && !schedulingRequests.isEmpty()) { SchedulerApplicationAttempt appAttempt = scheduler.getApplicationAttempt(appAttemptId); String queueName = null; if(appAttempt != null) { queueName = appAttempt.getQueueName(); } Resource maxAllocation = scheduler.getMaximumResourceCapability(queueName); // Normalize the Requests before dispatching schedulingRequests.forEach(req -> { Resource reqResource = req.getResourceSizing().getResources(); req.getResourceSizing().setResources( this.scheduler.getNormalizedResource(reqResource, maxAllocation)); }); this.placementDispatcher.dispatch(new BatchedRequests(iteratorType, appAttemptId.getApplicationId(), schedulingRequests, 1)); } }