private CapacitySchedulerContext mockCSContext( CapacitySchedulerConfiguration csConf, Resource clusterResource) { CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); when(csContext.getConfiguration()).thenReturn(csConf); when(csContext.getConf()).thenReturn(new YarnConfiguration()); when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); when(csContext.getClusterResource()).thenReturn(clusterResource); when(csContext.getMinimumResourceCapability()).thenReturn( Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(2 * GB, 2)); return csContext; }
@Before public void setUp() throws Exception { rmContext = TestUtils.getMockRMContext(); conf = new YarnConfiguration(); csConf = new CapacitySchedulerConfiguration(); csContext = mock(CapacitySchedulerContext.class); when(csContext.getConf()).thenReturn(conf); when(csContext.getConfiguration()).thenReturn(csConf); when(csContext.getMinimumResourceCapability()).thenReturn( Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(16*GB, 32)); when(csContext.getClusterResource()). thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32)); when(csContext.getApplicationComparator()). thenReturn(CapacityScheduler.applicationComparator); when(csContext.getQueueComparator()). thenReturn(CapacityScheduler.queueComparator); when(csContext.getResourceCalculator()). thenReturn(resourceComparator); when(csContext.getRMContext()).thenReturn(rmContext); }
public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, queueName, parent, old); this.scheduler = cs; this.activeUsersManager = new ActiveUsersManager(metrics); if(LOG.isDebugEnabled()) { LOG.debug("LeafQueue:" + " name=" + queueName + ", fullname=" + getQueuePath()); } Comparator<FiCaSchedulerApp> applicationComparator = cs.getApplicationComparator(); this.pendingApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator); this.activeApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator); setupQueueConfigs(cs.getClusterResource()); }
public ParentQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, queueName, parent, old); this.scheduler = cs; this.queueComparator = cs.getQueueComparator(); this.rootQueue = (parent == null); float rawCapacity = cs.getConfiguration().getNonLabeledQueueCapacity(getQueuePath()); if (rootQueue && (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { throw new IllegalArgumentException("Illegal " + "capacity of " + rawCapacity + " for queue " + queueName + ". Must be " + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE); } this.childQueues = new TreeSet<CSQueue>(queueComparator); setupQueueConfigs(cs.getClusterResource()); LOG.info("Initialized parent-queue " + queueName + " name=" + queueName + ", fullname=" + getQueuePath()); }
public LeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, configuration, queueName, parent, old); this.scheduler = cs; this.usersManager = new UsersManager(metrics, this, labelManager, scheduler, resourceCalculator); // One time initialization is enough since it is static ordering policy this.pendingOrderingPolicy = new FifoOrderingPolicyForPendingApps(); if(LOG.isDebugEnabled()) { LOG.debug("LeafQueue:" + " name=" + queueName + ", fullname=" + getQueuePath()); } setupQueueConfigs(cs.getClusterResource(), configuration); }
public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this.labelManager = cs.getRMContext().getNodeLabelManager(); this.parent = parent; this.queueName = queueName; this.resourceCalculator = cs.getResourceCalculator(); // must be called after parent and queueName is set this.metrics = old != null ? old.getMetrics() : QueueMetrics.forQueue(getQueuePath(), parent, cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); this.csContext = cs; this.minimumAllocation = csContext.getMinimumResourceCapability(); // initialize ResourceUsage queueUsage = new ResourceUsage(); queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath()); // initialize QueueCapacities queueCapacities = new QueueCapacities(parent == null); }
@Override public void recoverContainer(Resource clusterResource, SchedulerApplicationAttempt attempt, RMContainer rmContainer) { if (rmContainer.getState().equals(RMContainerState.COMPLETED)) { return; } if (rmContainer.getExecutionType() != ExecutionType.GUARANTEED) { return; } // Careful! Locking order is important! try { writeLock.lock(); FiCaSchedulerNode node = scheduler.getNode( rmContainer.getContainer().getNodeId()); allocateResource(clusterResource, rmContainer.getContainer().getResource(), node.getPartition()); } finally { writeLock.unlock(); } if (parent != null) { parent.recoverContainer(clusterResource, attempt, rmContainer); } }
@Override public synchronized void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { // Sanity check if (!(newlyParsedQueue instanceof ReservationQueue) || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) { throw new IOException("Trying to reinitialize " + getQueuePath() + " from " + newlyParsedQueue.getQueuePath()); } super.reinitialize(newlyParsedQueue, clusterResource); CSQueueUtils.updateQueueStatistics( parent.schedulerContext.getResourceCalculator(), newlyParsedQueue, parent, parent.schedulerContext.getClusterResource(), parent.schedulerContext.getMinimumResourceCapability()); updateQuotas(parent.getUserLimitForReservation(), parent.getUserLimitFactor(), parent.getMaxApplicationsForReservations(), parent.getMaxApplicationsPerUserForReservation()); }
scheduler.getNumClusterNodes()); return (Math.min(scheduler.getNumClusterNodes(), (requiredContainers * localityWaitFactor)) < missedOpportunities); Math.min(scheduler.getNumClusterNodes(), getNodeLocalityDelay()) < missedOpportunities );
csContext.getConfiguration().getAccessibleNodeLabels(getQueuePath()); this.defaultLabelExpression = csContext.getConfiguration() .getDefaultNodeLabelExpression(getQueuePath()); csContext.getConfiguration().getMaximumAllocationPerQueue( getQueuePath()); authorizer = YarnAuthorizationProvider.getInstance(csContext.getConf()); this.state = csContext.getConfiguration().getState(getQueuePath()); this.acls = csContext.getConfiguration().getAcls(getQueuePath()); this.reservationsContinueLooking = csContext.getConfiguration() .getReservationContinueLook();
public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this(cs, cs.getConfiguration(), queueName, parent, old); }
a.submitApplicationAttempt(app_1, user_0); when(csContext.getNode(node_0.getNodeID())).thenReturn(node_0); when(csContext.getNode(node_1.getNodeID())).thenReturn(node_1); when(csContext.getNumClusterNodes()).thenReturn(2);
@Override public void reinitializeQueues(CapacitySchedulerConfiguration newConf) throws IOException { // Parse new queues Map<String, CSQueue> newQueues = new HashMap<>(); CSQueue newRoot = parseQueue(this.csContext, newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP); // When failing over, if using configuration store, don't validate queue // hierarchy since queues can be removed without being STOPPED. if (!csContext.isConfigurationMutable() || csContext.getRMContext().getHAServiceState() != HAServiceProtocol.HAServiceState.STANDBY) { // Ensure queue hierarchy in the new XML file is proper. validateQueueHierarchy(queues, newQueues); } // Add new queues and delete OldQeueus only after validation. updateQueues(queues, newQueues); // Re-configure queues root.reinitialize(newRoot, this.csContext.getClusterResource()); setQueueAcls(authorizer, appPriorityACLManager, queues); // Re-calculate headroom for active applications Resource clusterResource = this.csContext.getClusterResource(); root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); labelManager.reinitializeQueueLabels(getQueueToLabels()); this.queueStateManager.initialize(this); }
public void reinitializeFromTemplate(AutoCreatedLeafQueueConfig leafQueueTemplate) throws SchedulerDynamicEditException, IOException { try { writeLock.lock(); // TODO: // reinitialize only capacities for now since 0 capacity updates // can cause // abs capacity related config computations to be incorrect if we go // through reinitialize QueueCapacities capacities = leafQueueTemplate.getQueueCapacities(); //update abs capacities setupConfigurableCapacities(capacities); //reset capacities for the leaf queue mergeCapacities(capacities); //update queue used capacity for all the node labels CSQueueUtils.updateQueueStatistics(resourceCalculator, csContext.getClusterResource(), this, labelManager, null); //activate applications if any are pending activateApplications(); } finally { writeLock.unlock(); } }
when(csContext.getClusterResource()) .thenReturn(Resources.createResource(16 * GB, 32)); Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
queueUsage.getUsed(partition), maxResource)) { RMContainer toKillContainer = killableContainerIter.next(); FiCaSchedulerApp attempt = csContext.getApplicationAttempt( toKillContainer.getContainerId().getApplicationAttemptId()); FiCaSchedulerNode node = csContext.getNode( toKillContainer.getAllocatedNode()); if (null != attempt && null != node) {
100 * 32); CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource); when(csContext.getRMContext()).thenReturn(rmContext); csConf.setFloat( CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this.labelManager = cs.getRMContext().getNodeLabelManager(); this.parent = parent; this.queueName = queueName; this.resourceCalculator = cs.getResourceCalculator(); // must be called after parent and queueName is set this.metrics = old != null ? old.getMetrics() : QueueMetrics.forQueue(getQueuePath(), parent, cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); this.csContext = cs; this.minimumAllocation = csContext.getMinimumResourceCapability(); // initialize ResourceUsage queueUsage = new ResourceUsage(); queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath()); // initialize QueueCapacities queueCapacities = new QueueCapacities(parent == null); }
FiCaSchedulerNode nodeToUnreserve = scheduler.getNode(idToUnreserve); if (nodeToUnreserve == null) { LOG.error("node to unreserve doesn't exist, nodeid: " + idToUnreserve);
@Override public synchronized void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { // Sanity check if (!(newlyParsedQueue instanceof ReservationQueue) || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) { throw new IOException("Trying to reinitialize " + getQueuePath() + " from " + newlyParsedQueue.getQueuePath()); } super.reinitialize(newlyParsedQueue, clusterResource); CSQueueUtils.updateQueueStatistics( parent.schedulerContext.getResourceCalculator(), newlyParsedQueue, parent, parent.schedulerContext.getClusterResource(), parent.schedulerContext.getMinimumResourceCapability()); updateQuotas(parent.getUserLimitForReservation(), parent.getUserLimitFactor(), parent.getMaxApplicationsForReservations(), parent.getMaxApplicationsPerUserForReservation()); }