public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, queueName, parent, old); this.scheduler = cs; this.activeUsersManager = new ActiveUsersManager(metrics); if(LOG.isDebugEnabled()) { LOG.debug("LeafQueue:" + " name=" + queueName + ", fullname=" + getQueuePath()); } Comparator<FiCaSchedulerApp> applicationComparator = cs.getApplicationComparator(); this.pendingApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator); this.activeApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator); setupQueueConfigs(cs.getClusterResource()); }
public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, queueName, parent, old); this.scheduler = cs; this.activeUsersManager = new ActiveUsersManager(metrics); if(LOG.isDebugEnabled()) { LOG.debug("LeafQueue:" + " name=" + queueName + ", fullname=" + getQueuePath()); } Comparator<FiCaSchedulerApp> applicationComparator = cs.getApplicationComparator(); this.pendingApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator); this.activeApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator); setupQueueConfigs(cs.getClusterResource()); }
.getQueuePath().equals(getQueuePath())) { throw new IOException( "Trying to reinitialize " + getQueuePath() + " from " + newlyParsedQueue.getQueuePath()); throw new IOException("Trying to reinitialize " + getQueuePath() + " the maximum allocation size can not be decreased!" + " Current setting: " + oldMax + ", trying to set it to: "
public LeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, configuration, queueName, parent, old); this.scheduler = cs; this.usersManager = new UsersManager(metrics, this, labelManager, scheduler, resourceCalculator); // One time initialization is enough since it is static ordering policy this.pendingOrderingPolicy = new FifoOrderingPolicyForPendingApps(); if(LOG.isDebugEnabled()) { LOG.debug("LeafQueue:" + " name=" + queueName + ", fullname=" + getQueuePath()); } setupQueueConfigs(cs.getClusterResource(), configuration); }
@Override public synchronized void reinitialize( CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { // Sanity check if (!(newlyParsedQueue instanceof LeafQueue) || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) { throw new IOException("Trying to reinitialize " + getQueuePath() + " from " + newlyParsedQueue.getQueuePath()); } LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue; // don't allow the maximum allocation to be decreased in size // since we have already told running AM's the size Resource oldMax = getMaximumAllocation(); Resource newMax = newlyParsedLeafQueue.getMaximumAllocation(); if (newMax.getMemory() < oldMax.getMemory() || newMax.getVirtualCores() < oldMax.getVirtualCores()) { throw new IOException( "Trying to reinitialize " + getQueuePath() + " the maximum allocation size can not be decreased!" + " Current setting: " + oldMax + ", trying to set it to: " + newMax); } setupQueueConfigs(clusterResource); // queue metrics are updated, more resource may be available // activate the pending applications if possible activateApplications(); }
@Override public synchronized void reinitialize( CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { // Sanity check if (!(newlyParsedQueue instanceof LeafQueue) || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) { throw new IOException("Trying to reinitialize " + getQueuePath() + " from " + newlyParsedQueue.getQueuePath()); } LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue; // don't allow the maximum allocation to be decreased in size // since we have already told running AM's the size Resource oldMax = getMaximumAllocation(); Resource newMax = newlyParsedLeafQueue.getMaximumAllocation(); if (newMax.getMemory() < oldMax.getMemory() || newMax.getVirtualCores() < oldMax.getVirtualCores()) { throw new IOException( "Trying to reinitialize " + getQueuePath() + " the maximum allocation size can not be decreased!" + " Current setting: " + oldMax + ", trying to set it to: " + newMax); } setupQueueConfigs(clusterResource); // queue metrics are updated, more resource may be available // activate the pending applications if possible activateApplications(); }
String msg = "Queue " + getQueuePath() + " is STOPPED. Cannot accept submission of application: " + applicationId; "Queue " + getQueuePath() + " already has " + getNumApplications() + " applications," + " cannot accept submission of application: " + applicationId; String msg = "Queue " + getQueuePath() + " already has " + user .getTotalApplications() + " applications from user " + userName + " cannot accept submission of application: " + applicationId;
&& !hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { throw new AccessControlException("User " + userName + " cannot submit" + " applications to queue " + getQueuePath()); String msg = "Queue " + getQueuePath() + " is STOPPED. Cannot accept submission of application: " + applicationId; LOG.info(msg); String msg = "Queue " + getQueuePath() + " already has " + getNumApplications() + " applications," + " cannot accept submission of application: " + applicationId; String msg = "Queue " + getQueuePath() + " already has " + user.getTotalApplications() + " applications from user " + userName +
&& !hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { throw new AccessControlException("User " + userName + " cannot submit" + " applications to queue " + getQueuePath()); String msg = "Queue " + getQueuePath() + " is STOPPED. Cannot accept submission of application: " + applicationId; LOG.info(msg); String msg = "Queue " + getQueuePath() + " already has " + getNumApplications() + " applications," + " cannot accept submission of application: " + applicationId; String msg = "Queue " + getQueuePath() + " already has " + user.getTotalApplications() + " applications from user " + userName +
userLimit = conf.getUserLimit(getQueuePath()); userLimitFactor = conf.getUserLimitFactor(getQueuePath()); maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); if (maxApplications < 0) { int maxSystemApps = conf.getMaximumSystemApplications(); conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
conf.<FiCaSchedulerApp>getAppOrderingPolicy(getQueuePath())); usersManager.setUserLimit(conf.getUserLimit(getQueuePath())); usersManager.setUserLimitFactor(conf.getUserLimitFactor(getQueuePath())); maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); if (maxApplications < 0) { int maxGlobalPerQueueApps = schedConf getQueuePath()); priorityAcls = conf.getPriorityAcls(getQueuePath(), scheduler.getMaxClusterLevelAppPriority()); conf.getDefaultApplicationPriorityConfPerQueue(getQueuePath())); conf.getMaximumLifetimePerQueue((getQueuePath())); defaultApplicationLifetime = conf.getDefaultLifetimePerQueue((getQueuePath())); if (defaultApplicationLifetime > maxApplicationLifetime) { throw new YarnRuntimeException( int queueUL = Math.min(100, conf.getUserLimit(getQueuePath())); for (Entry<String, Float> e : getUserWeights().entrySet()) { float val = e.getValue().floatValue(); + "\" must be between 0 and" + " 100 / " + queueUL + " (= " + 100.0f/queueUL + ", the number of concurrent active users in " + getQueuePath() + ")");
private void updateSchedulerHealthForCompletedContainer( RMContainer rmContainer, ContainerStatus containerStatus) { // Update SchedulerHealth for released / preempted container SchedulerHealth schedulerHealth = csContext.getSchedulerHealth(); if (null == schedulerHealth) { // Only do update if we have schedulerHealth return; } if (containerStatus.getExitStatus() == ContainerExitStatus.PREEMPTED) { schedulerHealth.updatePreemption(Time.now(), rmContainer.getAllocatedNode(), rmContainer.getContainerId(), getQueuePath()); schedulerHealth.updateSchedulerPreemptionCounts(1); } else { schedulerHealth.updateRelease(csContext.getLastNodeUpdateTime(), rmContainer.getAllocatedNode(), rmContainer.getContainerId(), getQueuePath()); } }
userLimit = conf.getUserLimit(getQueuePath()); userLimitFactor = conf.getUserLimitFactor(getQueuePath()); maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); if (maxApplications < 0) { int maxSystemApps = conf.getMaximumSystemApplications(); conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); int expectedMaxApps = (int) (long)CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, (long)csConf.getMaximumApplicationMasterResourcePerQueuePercent( queue.getQueuePath()) ); queue.getQueuePath() + ".maximum-am-resource-percent", 0.5f); queue.getQueuePath()) ); queue.getQueuePath() + ".maximum-applications", 9999); assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); assertEquals(9999, queue.getMaxApplications());
csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); int expectedMaxApps = (int) (long)CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, (long)csConf.getMaximumApplicationMasterResourcePerQueuePercent( queue.getQueuePath()) ); csConf.setFloat(PREFIX + queue.getQueuePath() + ".maximum-am-resource-percent", 0.5f); queue.getQueuePath()) ); csConf.setInt(PREFIX + queue.getQueuePath() + ".maximum-applications", 9999); assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); assertEquals(9999, queue.getMaxApplications());
schedulingMode)) { if (LOG.isDebugEnabled()) { LOG.debug("Skip this queue=" + getQueuePath() + ", because it doesn't need more resource, schedulingMode=" + schedulingMode.name() + " node-partition=" + candidates
updatedContainer, application.getCSLeafQueue().getQueuePath()); assignment.getAssignmentInformation().incrReservations(); Resources.addTo(assignment.getAssignmentInformation().getReserved(), updatedContainer, application.getCSLeafQueue().getQueuePath()); assignment.getAssignmentInformation().incrAllocations(); Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
csConf.setUserLimit(a.getQueuePath(), 50); csConf.setFloat("yarn.scheduler.capacity." + a.getQueuePath() + ".user-settings.user_0." + CapacitySchedulerConfiguration.USER_WEIGHT, 1.5f);