void addAMResourceUsage(Resource amResource) { if (amResource != null) { Resources.addTo(amResourceUsage, amResource); getMetrics().setAMResourceUsage(amResourceUsage); } }
public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); this.context = scheduler.getContext(); this.lastTimeAtMinShare = scheduler.getClock().getTime(); activeUsersManager = new ActiveUsersManager(getMetrics()); amResourceUsage = Resource.newInstance(0, 0); getMetrics().setAMResourceUsage(amResourceUsage); }
/** * Check whether this queue can run the Application Master under the * maxAMShare limit. * * @param amResource resources required to run the AM * @return true if this queue can run */ public boolean canRunAppAM(Resource amResource) { if (Math.abs(maxAMShare - -1.0f) < 0.0001) { return true; } Resource maxAMResource = computeMaxAMResource(); getMetrics().setMaxAMShare(maxAMResource); Resource ifRunAMResource = Resources.add(amResourceUsage, amResource); return Resources.fitsIn(ifRunAMResource, maxAMResource); }
/** * Removes the given app from this queue. * @return whether or not the app was runnable */ boolean removeApp(FSAppAttempt app) { boolean runnable = false; // Remove app from runnable/nonRunnable list while holding the write lock writeLock.lock(); try { runnable = runnableApps.remove(app); if (!runnable) { // removeNonRunnableApp acquires the write lock again, which is fine if (!removeNonRunnableApp(app)) { throw new IllegalStateException("Given app to remove " + app + " does not exist in queue " + this); } } } finally { writeLock.unlock(); } // Update AM resource usage if needed. If isAMRunning is true, we're not // running an unmanaged AM. if (runnable && app.isAmRunning()) { Resources.subtractFrom(amResourceUsage, app.getAMResource()); getMetrics().setAMResourceUsage(amResourceUsage); } decUsedResource(app.getResourceUsage()); return runnable; }
public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); this.lastTimeAtMinShare = scheduler.getClock().getTime(); this.lastTimeAtFairShareThreshold = scheduler.getClock().getTime(); activeUsersManager = new ActiveUsersManager(getMetrics()); amResourceUsage = Resource.newInstance(0, 0); }
public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); this.lastTimeAtMinShare = scheduler.getClock().getTime(); this.lastTimeAtFairShareThreshold = scheduler.getClock().getTime(); activeUsersManager = new ActiveUsersManager(getMetrics()); amResourceUsage = Resource.newInstance(0, 0); }
new SchedulerApplication<FSAppAttempt>(queue, user); applications.put(applicationId, application); queue.getMetrics().submitApp(user);
new SchedulerApplication<FSAppAttempt>(queue, user); applications.put(applicationId, application); queue.getMetrics().submitApp(user);
queue.getMetrics().submitAppAttempt(user);
new SchedulerApplication<FSAppAttempt>(queue, user); applications.put(applicationId, application); queue.getMetrics().submitApp(user);
queue.getMetrics().submitAppAttempt(user);
queue.getMetrics().submitAppAttempt(user);
@Test public void testUpdateDemand() { conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); String queueName = "root.queue1"; FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null); schedulable.setMaxShare(new ConfigurableResource(maxResource)); assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE); assertEquals(schedulable.getMetrics().getSchedulingPolicy(), SchedulingPolicy.DEFAULT_POLICY.getName()); FSAppAttempt app = mock(FSAppAttempt.class); Mockito.when(app.getDemand()).thenReturn(maxResource); Mockito.when(app.getResourceUsage()).thenReturn(Resources.none()); schedulable.addApp(app, true); schedulable.addApp(app, true); schedulable.updateDemand(); assertTrue("Demand is greater than max allowed ", Resources.equals(schedulable.getDemand(), maxResource)); }
greedyApp.getQueue().getMetrics().getAggregatePreemptedContainers());
assertEquals("QueueFSZeroWithMax's maximum AM resource should be " + "maxShare * maxAMShare", expectedAMResource.getMemorySize(), queueFSZeroWithMax.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " + "maxShare * maxAMShare", expectedAMResource.getVirtualCores(), queueFSZeroWithMax.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", amResource.getMemorySize(), queueFSZeroWithMax.getMetrics().getAMResourceUsageMB()); + " available resource * maxAMShare", expectedAMResource.getMemorySize(), queueFSZeroWithAVL.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", expectedAMResource.getVirtualCores(), queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", amResource.getMemorySize(), queueFSZeroWithAVL.getMetrics().getAMResourceUsageMB()); assertEquals("QueueFSNonZero's maximum AM resource should be " + " fair share * maxAMShare", expectedAMResource.getMemorySize(), queueFSNonZero.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSNonZero's maximum AM resource should be " + " fair share * maxAMShare", expectedAMResource.getVirtualCores(), queueFSNonZero.getMetrics().getMaxAMShareVCores());
FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3", true); assertEquals(capacity / 2, queue1.getFairShare().getMemory()); assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB()); assertEquals(capacity / 2, queue1.getSteadyFairShare().getMemory()); assertEquals(capacity / 2, queue1.getMetrics().getSteadyFairShareMB()); assertEquals(capacity / 4, queue2.getFairShare().getMemory()); assertEquals(capacity / 4, queue2.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue2.getSteadyFairShare().getMemory()); assertEquals(capacity / 4, queue2.getMetrics().getSteadyFairShareMB()); assertEquals(capacity / 4, queue3.getFairShare().getMemory()); assertEquals(capacity / 4, queue3.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue3.getSteadyFairShare().getMemory()); assertEquals(capacity / 4, queue3.getMetrics().getSteadyFairShareMB());
FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3", true); assertEquals(capacity / 2, queue1.getFairShare().getMemorySize()); assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB()); assertEquals(capacity / 2, queue1.getSteadyFairShare().getMemorySize()); assertEquals(capacity / 2, queue1.getMetrics().getSteadyFairShareMB()); assertEquals(capacity / 4, queue2.getFairShare().getMemorySize()); assertEquals(capacity / 4, queue2.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue2.getSteadyFairShare().getMemorySize()); assertEquals(capacity / 4, queue2.getMetrics().getSteadyFairShareMB()); assertEquals(capacity / 4, queue3.getFairShare().getMemorySize()); assertEquals(capacity / 4, queue3.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue3.getSteadyFairShare().getMemorySize()); assertEquals(capacity / 4, queue3.getMetrics().getSteadyFairShareMB());
@Test public void testSimpleFairShareCalculation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Have two queues which want entire cluster capacity createSchedulingRequest(10 * 1024, "queue1", "user1"); createSchedulingRequest(10 * 1024, "queue2", "user1"); createSchedulingRequest(10 * 1024, "root.default", "user1"); scheduler.update(); scheduler.getQueueManager().getRootQueue() .setSteadyFairShare(scheduler.getClusterResource()); scheduler.getQueueManager().getRootQueue().recomputeSteadyShares(); Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); // Divided three ways - between the two queues and the default queue for (FSLeafQueue p : queues) { assertEquals(3414, p.getFairShare().getMemorySize()); assertEquals(3414, p.getMetrics().getFairShareMB()); assertEquals(3414, p.getSteadyFairShare().getMemorySize()); assertEquals(3414, p.getMetrics().getSteadyFairShareMB()); } }
@Test public void testSimpleFairShareCalculation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Have two queues which want entire cluster capacity createSchedulingRequest(10 * 1024, "queue1", "user1"); createSchedulingRequest(10 * 1024, "queue2", "user1"); createSchedulingRequest(10 * 1024, "root.default", "user1"); scheduler.update(); scheduler.getQueueManager().getRootQueue() .setSteadyFairShare(scheduler.getClusterResource()); scheduler.getQueueManager().getRootQueue().recomputeSteadyShares(); Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); // Divided three ways - between the two queues and the default queue for (FSLeafQueue p : queues) { assertEquals(3414, p.getFairShare().getMemory()); assertEquals(3414, p.getMetrics().getFairShareMB()); assertEquals(3414, p.getSteadyFairShare().getMemory()); assertEquals(3414, p.getMetrics().getSteadyFairShareMB()); } }