return; CSQueue queue = (CSQueue) application.getQueue(); new FiCaSchedulerApp(applicationAttemptId, application.getUser(), queue, queue.getActiveUsersManager(), rmContext); if (transferStateFromPreviousAttempt) { attempt.transferStateFromPreviousAttempt(application .getCurrentAppAttempt()); application.setCurrentAppAttempt(attempt); queue.submitApplicationAttempt(attempt, application.getUser()); LOG.info("Added Application Attempt " + applicationAttemptId + " to scheduler from user " + application.getUser() + " in queue " + queue.getQueueName()); if (isAttemptRecovering) {
@VisibleForTesting public synchronized void addApplication(ApplicationId applicationId, String queue, String user, boolean isAppRecovering) { SchedulerApplication<FifoAppAttempt> application = new SchedulerApplication<>(DEFAULT_QUEUE, user); applications.put(applicationId, application); metrics.submitApp(user); LOG.info("Accepted application " + applicationId + " from user: " + user + ", currently num of applications: " + applications.size()); if (isAppRecovering) { if (LOG.isDebugEnabled()) { LOG.debug(applicationId + " is recovering. Skip notifying APP_ACCEPTED"); } } else { rmContext.getDispatcher().getEventHandler() .handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); } }
private synchronized void doneApplication(ApplicationId applicationId, RMAppState finalState) { SchedulerApplication<FiCaSchedulerApp> application = applications.get(applicationId); if (application == null){ // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps, // ignore it. LOG.warn("Couldn't find application " + applicationId); return; } CSQueue queue = (CSQueue) application.getQueue(); if (!(queue instanceof LeafQueue)) { LOG.error("Cannot finish application " + "from non-leaf queue: " + queue.getQueueName()); } else { queue.finishApplication(applicationId, application.getUser()); } application.stop(finalState); applications.remove(applicationId); }
private synchronized void doneApplication(ApplicationId applicationId, RMAppState finalState) { SchedulerApplication<FiCaSchedulerApp> application = applications.get(applicationId); if (application == null){ LOG.warn("Couldn't find application " + applicationId); return; } // Inform the activeUsersManager activeUsersManager.deactivateApplication(application.getUser(), applicationId); application.stop(finalState); applications.remove(applicationId); }
@VisibleForTesting public void clearPendingContainerCache() { for (SchedulerApplication<T> app : applications.values()) { T attempt = app.getCurrentAppAttempt(); if (attempt != null) { for (ContainerId containerId : attempt.getPendingRelease()) { RMAuditLogger.logFailure(app.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app " + "or with invalid id.", attempt.getApplicationId(), containerId, null); } attempt.getPendingRelease().clear(); } } }
public T getApplicationAttempt(ApplicationAttemptId applicationAttemptId) { SchedulerApplication<T> app = applications.get(applicationAttemptId.getApplicationId()); return app == null ? null : app.getCurrentAppAttempt(); }
@Override public void preValidateMoveApplication(ApplicationId appId, String newQueue) throws YarnException { writeLock.lock(); try { SchedulerApplication<FSAppAttempt> app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); } FSAppAttempt attempt = app.getCurrentAppAttempt(); // To serialize with FairScheduler#allocate, synchronize on app attempt attempt.getWriteLock().lock(); try { FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); String destQueueName = handleMoveToPlanQueue(newQueue); FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); if (targetQueue == null) { throw new YarnException("Target queue " + newQueue + " not found or is not a leaf queue."); } if (oldQueue.isRunnableApp(attempt)) { verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); } } finally { attempt.getWriteLock().unlock(); } } finally { writeLock.unlock(); } }
throw new YarnException("App to be moved " + appId + " not found."); String sourceQueueName = application.getQueue().getQueueName(); LeafQueue source = this.queueManager.getAndCheckLeafQueue(sourceQueueName); LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); String user = application.getUser(); try { dest.submitApplication(appId, user, destQueueName); FiCaSchedulerApp app = application.getCurrentAppAttempt(); if (app != null) { application.setQueue(dest); LOG.info("App: " + appId + " successfully moved from " + sourceQueueName + " to: " + destQueueName);
private void checkCSQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) throws Exception { checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource, numContainers); LeafQueue queue = (LeafQueue) app.getQueue(); Resource availableResources = Resources.subtract(queueResource, usedResource); // ************ check app headroom **************** SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt(); assertEquals(availableResources, schedulerAttempt.getHeadroom()); // ************* check Queue metrics ************ QueueMetrics queueMetrics = queue.getMetrics(); assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemorySize(), availableResources.getVirtualCores(), usedResource.getMemorySize(), usedResource.getVirtualCores()); // ************ check user metrics *********** QueueMetrics userMetrics = queueMetrics.getUserMetrics(app.getUser()); assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemorySize(), availableResources.getVirtualCores(), usedResource.getMemorySize(), usedResource.getVirtualCores()); }
SchedulerApplication<FiCaSchedulerApp> application = applications.get(appAttemptId.getApplicationId()); String user = application.getUser(); .getCurrentAppAttempt()); application.setCurrentAppAttempt(schedulerApp); + " to scheduler from user " + application.getUser()); if (isAttemptRecovering) { if (LOG.isDebugEnabled()) {
throw new YarnException("App to be moved " + appId + " not found."); String sourceQueueName = application.getQueue().getQueueName(); this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(newQueue); LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); String user = application.getUser();
SchedulerApplication app = applications.get(appAddEvent.getApplicationAttemptId().getApplicationId()); appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue().getQueueName());
private RMContainer recoverAndCreateContainer(NMContainerStatus status, RMNode node) { Container container = Container.newInstance(status.getContainerId(), node.getNodeID(), node.getHttpAddress(), status.getAllocatedResource(), status.getPriority(), null); ApplicationAttemptId attemptId = container.getId().getApplicationAttemptId(); RMContainer rmContainer = new RMContainerImpl(container, attemptId, node.getNodeID(), applications.get(attemptId.getApplicationId()).getUser(), rmContext, status.getCreationTime()); return rmContainer; }
new MockRMApp(125, System.currentTimeMillis(), RMAppState.NEW); SchedulerApplication<FiCaSchedulerApp> application = new SchedulerApplication<FiCaSchedulerApp>(null, mockAPp.getUser()); .getCurrentAppAttempt().getPendingRelease().size(), 0); } finally { if (rm1 != null) {
app.setQueue(newQueue); newQueue.addApp(attempt, nowRunnable);
private void removeApplication(ApplicationId applicationId, RMAppState finalState) { SchedulerApplication<FSAppAttempt> application = applications.remove( applicationId); if (application == null) { LOG.warn("Couldn't find application " + applicationId); } else{ application.stop(finalState); } }
public T getApplicationAttempt(ApplicationAttemptId applicationAttemptId) { SchedulerApplication<T> app = applications.get(applicationAttemptId.getApplicationId()); return app == null ? null : app.getCurrentAppAttempt(); }
@Override public synchronized String moveApplication(ApplicationId appId, String queueName) throws YarnException { SchedulerApplication<FSAppAttempt> app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); } FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt(); // To serialize with FairScheduler#allocate, synchronize on app attempt synchronized (attempt) { FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); String destQueueName = handleMoveToPlanQueue(queueName); FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); if (targetQueue == null) { throw new YarnException("Target queue " + queueName + " not found or is not a leaf queue."); } if (targetQueue == oldQueue) { return oldQueue.getQueueName(); } if (oldQueue.isRunnableApp(attempt)) { verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); } executeMove(app, attempt, oldQueue, targetQueue); return targetQueue.getQueueName(); } }
private void checkCSQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) throws Exception { checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource, numContainers); LeafQueue queue = (LeafQueue) app.getQueue(); Resource availableResources = Resources.subtract(queueResource, usedResource); // ************ check app headroom **************** SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt(); assertEquals(availableResources, schedulerAttempt.getHeadroom()); // ************* check Queue metrics ************ QueueMetrics queueMetrics = queue.getMetrics(); assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(), availableResources.getVirtualCores(), usedResource.getMemory(), usedResource.getVirtualCores()); // ************ check user metrics *********** QueueMetrics userMetrics = queueMetrics.getUserMetrics(app.getUser()); assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(), availableResources.getVirtualCores(), usedResource.getMemory(), usedResource.getVirtualCores()); }
SchedulerApplication<FiCaSchedulerApp> application = applications.get(appAttemptId.getApplicationId()); String user = application.getUser(); .getCurrentAppAttempt()); application.setCurrentAppAttempt(schedulerApp); + " to scheduler from user " + application.getUser()); if (isAttemptRecovering) { if (LOG.isDebugEnabled()) {