@Override public Integer getValue() { return queue.getDemand().getMemory(); } }
@Override public Integer getValue() { return queue.getDemand().getVirtualCores(); } }
/** Convenient toString implementation for debugging. */ @Override public String toString() { return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]", getName(), getDemand(), getResourceUsage(), fairShare, getWeights()); }
/** Convenient toString implementation for debugging. */ @Override public String toString() { return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]", getName(), getDemand(), getResourceUsage(), fairShare, getWeight()); }
/** Convenient toString implementation for debugging. */ @Override public String toString() { return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]", getName(), getDemand(), getResourceUsage(), fairShare, getWeights()); }
@Override public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources writeLock.lock(); try { demand = Resources.createResource(0); for (FSQueue childQueue : childQueues) { childQueue.updateDemand(); Resource toAdd = childQueue.getDemand(); demand = Resources.add(demand, toAdd); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + childQueue.getName() + " " + toAdd + "; Total resource demand for " + getName() + " now " + demand); } } // Cap demand to maxShare to limit allocation to maxShare demand = Resources.componentwiseMin(demand, getMaxShare()); } finally { writeLock.unlock(); } if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + getMaxShare()); } }
/** * Dump scheduler state including states of all queues. */ private void dumpSchedulerState() { FSQueue rootQueue = queueMgr.getRootQueue(); Resource clusterResource = getClusterResource(); STATE_DUMP_LOG.debug( "FairScheduler state: Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); STATE_DUMP_LOG.debug(rootQueue.dumpState()); }
@Override public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources Resource maxRes = scheduler.getAllocationConfiguration() .getMaxResources(getName()); demand = Resources.createResource(0); for (FSQueue childQueue : childQueues) { childQueue.updateDemand(); Resource toAdd = childQueue.getDemand(); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + childQueue.getName() + " " + toAdd + "; Total resource consumption for " + getName() + " now " + demand); } demand = Resources.add(demand, toAdd); demand = Resources.componentwiseMin(demand, maxRes); if (Resources.equals(demand, maxRes)) { break; } } if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + maxRes); } }
@Override public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources Resource maxRes = scheduler.getAllocationConfiguration() .getMaxResources(getName()); demand = Resources.createResource(0); for (FSQueue childQueue : childQueues) { childQueue.updateDemand(); Resource toAdd = childQueue.getDemand(); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + childQueue.getName() + " " + toAdd + "; Total resource consumption for " + getName() + " now " + demand); } demand = Resources.add(demand, toAdd); demand = Resources.componentwiseMin(demand, maxRes); if (Resources.equals(demand, maxRes)) { break; } } if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + maxRes); } }
@Test public void testNormalizationUsingQueueMaximumAllocation() throws IOException { int queueMaxAllocation = 4096; conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println(" <queue name=\"queueA\">"); out.println(" <maxContainerAllocation>" + queueMaxAllocation + " mb 1 vcores" + "</maxContainerAllocation>"); out.println(" </queue>"); out.println(" <queue name=\"queueB\">"); out.println(" </queue>"); out.println("</allocations>"); out.close(); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); allocateAppAttempt("root.queueA", 1, queueMaxAllocation + 1024); allocateAppAttempt("root.queueB", 2, RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE + 1024); scheduler.update(); FSQueue queueToCheckA = scheduler.getQueueManager().getQueue("root.queueA"); FSQueue queueToCheckB = scheduler.getQueueManager().getQueue("root.queueB"); assertEquals(queueMaxAllocation, queueToCheckA.getDemand().getMemorySize()); assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, queueToCheckB.getDemand().getMemorySize()); }
/** * Recompute the internal variables used by the scheduler - per-job weights, * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ protected synchronized void update() { long start = getClock().getTime(); updateStarvationStats(); // Determine if any queues merit preemption FSQueue rootQueue = queueMgr.getRootQueue(); // Recursively update demands for all queues rootQueue.updateDemand(); rootQueue.setFairShare(clusterResource); // Recursively compute fair shares for all queues // and update metrics rootQueue.recomputeShares(); updateRootQueueMetrics(); if (LOG.isDebugEnabled()) { if (--updatesToSkipForDebug < 0) { updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; LOG.debug("Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); } } long duration = getClock().getTime() - start; fsOpDurations.addUpdateCallDuration(duration); }
/** * Recompute the internal variables used by the scheduler - per-job weights, * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ protected synchronized void update() { long start = getClock().getTime(); updateStarvationStats(); // Determine if any queues merit preemption FSQueue rootQueue = queueMgr.getRootQueue(); // Recursively update demands for all queues rootQueue.updateDemand(); rootQueue.setFairShare(clusterResource); // Recursively compute fair shares for all queues // and update metrics rootQueue.recomputeShares(); updateRootQueueMetrics(); if (LOG.isDebugEnabled()) { if (--updatesToSkipForDebug < 0) { updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; LOG.debug("Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); } } long duration = getClock().getTime() - start; fsOpDurations.addUpdateCallDuration(duration); }
queue.getMetrics().getMaxAMShareVCores())); usedResources = new ResourceInfo(queue.getResourceUsage()); demandResources = new ResourceInfo(queue.getDemand()); fractionMemUsed = (float)usedResources.getMemorySize() / clusterResources.getMemorySize();