@Override public Resource getMaximumResourceCapability(String queueName) { CSQueue queue = getQueue(queueName); if (queue == null) { LOG.error("Unknown queue: " + queueName); return getMaximumResourceCapability(); } if (!(queue instanceof LeafQueue)) { LOG.error("queue " + queueName + " is not an leaf queue"); return getMaximumResourceCapability(); } return ((LeafQueue)queue).getMaximumAllocation(); }
@Override public Resource getMaximumResourceCapability(String queueName) { CSQueue queue = getQueue(queueName); if (queue == null) { LOG.error("Unknown queue: " + queueName); return getMaximumResourceCapability(); } if (!(queue instanceof LeafQueue)) { LOG.error("queue " + queueName + " is not an leaf queue"); return getMaximumResourceCapability(); } return ((LeafQueue)queue).getMaximumAllocation(); }
Resource oldMax = getMaximumAllocation(); Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
@Override public Resource getMaximumResourceCapability(String queueName) { if(queueName == null || queueName.isEmpty()) { return getMaximumResourceCapability(); } CSQueue queue = getQueue(queueName); if (queue == null) { LOG.error("Unknown queue: " + queueName); return getMaximumResourceCapability(); } if (!(queue instanceof LeafQueue)) { LOG.error("queue " + queueName + " is not an leaf queue"); return getMaximumResourceCapability(); } // queue.getMaxAllocation returns *configured* maximum allocation. // getMaximumResourceCapability() returns maximum allocation considers // per-node maximum resources. So return (component-wise) min of the two. Resource queueMaxAllocation = ((LeafQueue)queue).getMaximumAllocation(); Resource clusterMaxAllocationConsiderNodeMax = getMaximumResourceCapability(); return Resources.componentwiseMin(queueMaxAllocation, clusterMaxAllocationConsiderNodeMax); }
boolean shouldAllocOrReserveNewContainer(FiCaSchedulerApp application, Priority priority, Resource required) { int requiredContainers = application.getTotalRequiredResources(priority); int reservedContainers = application.getNumReservedContainers(priority); int starvation = 0; if (reservedContainers > 0) { float nodeFactor = Resources.ratio( resourceCalculator, required, getMaximumAllocation() ); // Use percentage of node required to bias against large containers... // Protect against corner case where you need the whole node with // Math.min(nodeFactor, minimumAllocationFactor) starvation = (int)((application.getReReservations(priority) / (float)reservedContainers) * (1.0f - (Math.min(nodeFactor, getMinimumAllocationFactor()))) ); if (LOG.isDebugEnabled()) { LOG.debug("needsContainers:" + " app.#re-reserve=" + application.getReReservations(priority) + " reserved=" + reservedContainers + " nodeFactor=" + nodeFactor + " minAllocFactor=" + getMinimumAllocationFactor() + " starvation=" + starvation); } } return (((starvation + requiredContainers) - reservedContainers) > 0); }
boolean shouldAllocOrReserveNewContainer(FiCaSchedulerApp application, Priority priority, Resource required) { int requiredContainers = application.getTotalRequiredResources(priority); int reservedContainers = application.getNumReservedContainers(priority); int starvation = 0; if (reservedContainers > 0) { float nodeFactor = Resources.ratio( resourceCalculator, required, getMaximumAllocation() ); // Use percentage of node required to bias against large containers... // Protect against corner case where you need the whole node with // Math.min(nodeFactor, minimumAllocationFactor) starvation = (int)((application.getReReservations(priority) / (float)reservedContainers) * (1.0f - (Math.min(nodeFactor, getMinimumAllocationFactor()))) ); if (LOG.isDebugEnabled()) { LOG.debug("needsContainers:" + " app.#re-reserve=" + application.getReReservations(priority) + " reserved=" + reservedContainers + " nodeFactor=" + nodeFactor + " minAllocFactor=" + getMinimumAllocationFactor() + " starvation=" + starvation); } } return (((starvation + requiredContainers) - reservedContainers) > 0); }
@Override public synchronized void reinitialize( CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { // Sanity check if (!(newlyParsedQueue instanceof LeafQueue) || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) { throw new IOException("Trying to reinitialize " + getQueuePath() + " from " + newlyParsedQueue.getQueuePath()); } LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue; // don't allow the maximum allocation to be decreased in size // since we have already told running AM's the size Resource oldMax = getMaximumAllocation(); Resource newMax = newlyParsedLeafQueue.getMaximumAllocation(); if (newMax.getMemory() < oldMax.getMemory() || newMax.getVirtualCores() < oldMax.getVirtualCores()) { throw new IOException( "Trying to reinitialize " + getQueuePath() + " the maximum allocation size can not be decreased!" + " Current setting: " + oldMax + ", trying to set it to: " + newMax); } setupQueueConfigs(clusterResource); // queue metrics are updated, more resource may be available // activate the pending applications if possible activateApplications(); }
@Override public synchronized void reinitialize( CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { // Sanity check if (!(newlyParsedQueue instanceof LeafQueue) || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) { throw new IOException("Trying to reinitialize " + getQueuePath() + " from " + newlyParsedQueue.getQueuePath()); } LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue; // don't allow the maximum allocation to be decreased in size // since we have already told running AM's the size Resource oldMax = getMaximumAllocation(); Resource newMax = newlyParsedLeafQueue.getMaximumAllocation(); if (newMax.getMemory() < oldMax.getMemory() || newMax.getVirtualCores() < oldMax.getVirtualCores()) { throw new IOException( "Trying to reinitialize " + getQueuePath() + " the maximum allocation size can not be decreased!" + " Current setting: " + oldMax + ", trying to set it to: " + newMax); } setupQueueConfigs(clusterResource); // queue metrics are updated, more resource may be available // activate the pending applications if possible activateApplications(); }
boolean shouldAllocOrReserveNewContainer( SchedulerRequestKey schedulerKey, Resource required) { int requiredContainers = application.getOutstandingAsksCount(schedulerKey); int reservedContainers = application.getNumReservedContainers(schedulerKey); int starvation = 0; if (reservedContainers > 0) { float nodeFactor = Resources.ratio( rc, required, application.getCSLeafQueue().getMaximumAllocation()); // Use percentage of node required to bias against large containers... // Protect against corner case where you need the whole node with // Math.min(nodeFactor, minimumAllocationFactor) starvation = (int) ((application.getReReservations(schedulerKey) / (float) reservedContainers) * (1.0f - (Math.min( nodeFactor, application.getCSLeafQueue() .getMinimumAllocationFactor())))); if (LOG.isDebugEnabled()) { LOG.debug("needsContainers:" + " app.#re-reserve=" + application.getReReservations(schedulerKey) + " reserved=" + reservedContainers + " nodeFactor=" + nodeFactor + " minAllocFactor=" + application.getCSLeafQueue().getMinimumAllocationFactor() + " starvation=" + starvation); } } return (((starvation + requiredContainers) - reservedContainers) > 0); }
((LeafQueue) queueA1).getMaximumAllocation().getMemory()); assertEquals("queue A1 max allocation vcores", 4, ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores()); assertEquals("queue A2 max allocation MB", 10240, ((LeafQueue) queueA2).getMaximumAllocation().getMemory()); assertEquals("queue A2 max allocation vcores", 10, ((LeafQueue) queueA2).getMaximumAllocation().getVirtualCores()); assertEquals("queue B2 max allocation MB", 10240, ((LeafQueue) queueB2).getMaximumAllocation().getMemory()); assertEquals("queue B2 max allocation vcores", 10, ((LeafQueue) queueB2).getMaximumAllocation().getVirtualCores()); cs.getMaximumResourceCapability().getVirtualCores()); assertEquals("queue A1 max MB allocation", 4096, ((LeafQueue) queueA1).getMaximumAllocation().getMemory()); assertEquals("queue A1 max vcores allocation", 4, ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores()); assertEquals("queue A2 max MB allocation", 12288, ((LeafQueue) queueA2).getMaximumAllocation().getMemory()); assertEquals("queue A2 max vcores allocation", 12, ((LeafQueue) queueA2).getMaximumAllocation().getVirtualCores()); assertEquals("queue B2 max MB allocation", 12288, ((LeafQueue) queueB2).getMaximumAllocation().getMemory()); assertEquals("queue B2 max vcores allocation", 12, ((LeafQueue) queueB2).getMaximumAllocation().getVirtualCores());
((LeafQueue) queueA1).getMaximumAllocation().getMemorySize()); assertEquals("queue A1 max allocation vcores", 4, ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores()); assertEquals("queue A2 max allocation MB", 10240, ((LeafQueue) queueA2).getMaximumAllocation().getMemorySize()); assertEquals("queue A2 max allocation vcores", 10, ((LeafQueue) queueA2).getMaximumAllocation().getVirtualCores()); assertEquals("queue B2 max allocation MB", 10240, ((LeafQueue) queueB2).getMaximumAllocation().getMemorySize()); assertEquals("queue B2 max allocation vcores", 10, ((LeafQueue) queueB2).getMaximumAllocation().getVirtualCores()); cs.getMaximumResourceCapability().getVirtualCores()); assertEquals("queue A1 max MB allocation", 4096, ((LeafQueue) queueA1).getMaximumAllocation().getMemorySize()); assertEquals("queue A1 max vcores allocation", 4, ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores()); assertEquals("queue A2 max MB allocation", 12288, ((LeafQueue) queueA2).getMaximumAllocation().getMemorySize()); assertEquals("queue A2 max vcores allocation", 12, ((LeafQueue) queueA2).getMaximumAllocation().getVirtualCores()); assertEquals("queue B2 max MB allocation", 12288, ((LeafQueue) queueB2).getMaximumAllocation().getMemorySize()); assertEquals("queue B2 max vcores allocation", 12, ((LeafQueue) queueB2).getMaximumAllocation().getVirtualCores());
CSQueue queueA1 = findQueue(queueA, A1); assertEquals("queue max allocation", ((LeafQueue) queueA1) .getMaximumAllocation().getMemory(), 8192);
CSQueue queueA1 = findQueue(queueA, A1); assertEquals("queue max allocation", ((LeafQueue) queueA1) .getMaximumAllocation().getMemorySize(), 8192);
when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(4*GB, 16)); when(a.getMaximumAllocation()).thenReturn( Resources.createResource(4*GB, 16)); when(a.getMinimumAllocationFactor()).thenReturn(0.25f); // 1G / 4G
when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(4*GB, 16)); when(a.getMaximumAllocation()).thenReturn( Resources.createResource(4*GB, 16)); when(a.getMinimumAllocationFactor()).thenReturn(0.25f); // 1G / 4G
CSQueue queueA1 = findQueue(queueA, A1); assertEquals("queue max allocation", ((LeafQueue) queueA1) .getMaximumAllocation().getMemory(), 4096); conf.getMaximumAllocation().getVirtualCores()); assertEquals("queue max allocation MB", 6144, ((LeafQueue) queueA1).getMaximumAllocation().getMemory()); assertEquals("queue max allocation vcores", 3, ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores()); assertEquals("max capability MB cluster", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
CSQueue queueA1 = findQueue(queueA, A1); assertEquals("queue max allocation", 4096, ((LeafQueue) queueA1) .getMaximumAllocation().getMemorySize()); ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); assertEquals("queue max allocation MB", 6144, ((LeafQueue) queueA1).getMaximumAllocation().getMemorySize()); assertEquals("queue max allocation vcores", 3, ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores()); assertEquals("max capability MB cluster", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,