@Override public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { return 0; } else { return scheduler.getRootQueueMetrics().getAvailableMB(); } } }
@Override public Integer getValue() { if(getRootQueueMetrics() == null) { return 0; } else { return getRootQueueMetrics().getAvailableMB(); } } }
/** * Compute the maximum resource AM can use. The value is the result of * multiplying FairShare and maxAMShare. If FairShare is zero, use * min(maxShare, available resource) instead to prevent zero value for * maximum AM resource since it forbids any job running in the queue. * * @return the maximum resource AM can use */ private Resource computeMaxAMResource() { Resource maxResource = Resources.clone(getFairShare()); if (maxResource.getMemorySize() == 0) { maxResource.setMemorySize( Math.min(scheduler.getRootQueueMetrics().getAvailableMB(), getMaxShare().getMemorySize())); } if (maxResource.getVirtualCores() == 0) { maxResource.setVirtualCores(Math.min( scheduler.getRootQueueMetrics().getAvailableVirtualCores(), getMaxShare().getVirtualCores())); } // Round up to allow AM to run when there is only one vcore on the cluster return Resources.multiplyAndRoundUp(maxResource, maxAMShare); }
private void verifyNoCalls() { assertFalse("Update thread should not have executed", scheduler.fsOpDurations.hasUpdateThreadRunChanged()); assertEquals("Scheduler queue memory should not have been updated", 0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals("Scheduler queue cpu should not have been updated", 0,scheduler.getRootQueueMetrics().getAvailableVirtualCores()); } }
private void verifyMetrics(QueueMetrics metrics, long availableMB, int availableVirtualCores, long allocatedMB, int allocatedVirtualCores, int allocatedContainers) { Assert.assertEquals(availableMB, metrics.getAvailableMB()); Assert.assertEquals(availableVirtualCores, metrics.getAvailableVirtualCores()); Assert.assertEquals(allocatedMB, metrics.getAllocatedMB()); Assert.assertEquals(allocatedVirtualCores, metrics.getAllocatedVirtualCores()); Assert.assertEquals(allocatedContainers, metrics.getAllocatedContainers()); }
private static void verifyMetrics(QueueMetrics metrics, long availableMB, int availableVirtualCores, long allocatedMB, int allocatedVirtualCores, int allocatedContainers) { Assert.assertEquals(availableMB, metrics.getAvailableMB()); Assert.assertEquals(availableVirtualCores, metrics.getAvailableVirtualCores()); Assert.assertEquals(allocatedMB, metrics.getAllocatedMB()); Assert.assertEquals(allocatedVirtualCores, metrics.getAllocatedVirtualCores()); Assert.assertEquals(allocatedContainers, metrics.getAllocatedContainers()); } }
assertMetric("containersPending", containersPending, metrics.getPendingContainers()); assertMetric("availableMB", availableMB, metrics.getAvailableMB()); assertMetric("activeApplications", activeApplications, metrics.getActiveApps());
assertMetric("containersPending", containersPending, metrics.getPendingContainers()); assertMetric("availableMB", availableMB, metrics.getAvailableMB()); assertMetric("activeApplications", activeApplications, metrics.getActiveApps());
assertEquals( (int)(node_0.getTotalResource().getMemory() * a.getCapacity()) - (1*GB), a.getMetrics().getAvailableMB());
@Test public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent addEvent = new NodeAddedSchedulerEvent(node1); scheduler.handle(addEvent); assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); NodeRemovedSchedulerEvent removeEvent = new NodeRemovedSchedulerEvent(node1); scheduler.handle(removeEvent); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); }
@Test public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent addEvent = new NodeAddedSchedulerEvent(node1); scheduler.handle(addEvent); assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); NodeRemovedSchedulerEvent removeEvent = new NodeRemovedSchedulerEvent(node1); scheduler.handle(removeEvent); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); }
scheduler.fsOpDurations.hasUpdateThreadRunChanged()); assertEquals("Root queue metrics memory does not have expected value", memory, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals("Root queue metrics cpu does not have expected value", vcores, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
public ClusterMetricsInfo(final ResourceManager rm) { ResourceScheduler rs = rm.getResourceScheduler(); QueueMetrics metrics = rs.getRootQueueMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); this.appsSubmitted = metrics.getAppsSubmitted(); this.appsCompleted = metrics.getAppsCompleted(); this.appsPending = metrics.getAppsPending(); this.appsRunning = metrics.getAppsRunning(); this.appsFailed = metrics.getAppsFailed(); this.appsKilled = metrics.getAppsKilled(); this.reservedMB = metrics.getReservedMB(); this.availableMB = metrics.getAvailableMB(); this.allocatedMB = metrics.getAllocatedMB(); this.reservedVirtualCores = metrics.getReservedVirtualCores(); this.availableVirtualCores = metrics.getAvailableVirtualCores(); this.allocatedVirtualCores = metrics.getAllocatedVirtualCores(); this.containersAllocated = metrics.getAllocatedContainers(); this.containersPending = metrics.getPendingContainers(); this.containersReserved = metrics.getReservedContainers(); this.totalMB = availableMB + allocatedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; this.activeNodes = clusterMetrics.getNumActiveNMs(); this.lostNodes = clusterMetrics.getNumLostNMs(); this.unhealthyNodes = clusterMetrics.getUnhealthyNMs(); this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs(); this.rebootedNodes = clusterMetrics.getNumRebootedNMs(); this.totalNodes = activeNodes + lostNodes + decommissionedNodes + rebootedNodes + unhealthyNodes; }
this.availableMB = metrics.getAvailableMB(); this.allocatedMB = metrics.getAllocatedMB();
private void assertMetrics(QueueMetrics qm, int appsSubmitted, int appsPending, int appsRunning, int appsCompleted, int allocatedContainers, int availableMB, int availableVirtualCores, int allocatedMB, int allocatedVirtualCores) { assertEquals(appsSubmitted, qm.getAppsSubmitted()); assertEquals(appsPending, qm.getAppsPending()); assertEquals(appsRunning, qm.getAppsRunning()); assertEquals(appsCompleted, qm.getAppsCompleted()); assertEquals(allocatedContainers, qm.getAllocatedContainers()); assertEquals(availableMB, qm.getAvailableMB()); assertEquals(availableVirtualCores, qm.getAvailableVirtualCores()); assertEquals(allocatedMB, qm.getAllocatedMB()); assertEquals(allocatedVirtualCores, qm.getAllocatedVirtualCores()); }
@Test(timeout = 50000) public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.setQueues("default", new String[] { "default" }); conf.setCapacity("default", 100); FifoScheduler fs = new FifoScheduler(); fs.init(conf); fs.start(); // mock rmContext to avoid NPE. RMContext context = mock(RMContext.class); fs.reinitialize(conf, null); fs.setRMContext(context); RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2"); RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2, "127.0.0.3"); fs.handle(new NodeAddedSchedulerEvent(n1)); fs.handle(new NodeAddedSchedulerEvent(n2)); fs.handle(new NodeUpdateSchedulerEvent(n1)); Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB()); // reconnect n1 with downgraded memory n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1, "127.0.0.2"); fs.handle(new NodeRemovedSchedulerEvent(n1)); fs.handle(new NodeAddedSchedulerEvent(n1)); fs.handle(new NodeUpdateSchedulerEvent(n1)); Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB()); fs.stop(); }
private void assertMetrics(QueueMetrics qm, int appsSubmitted, int appsPending, int appsRunning, int appsCompleted, int allocatedContainers, long availableMB, long availableVirtualCores, long allocatedMB, long allocatedVirtualCores) { assertEquals(appsSubmitted, qm.getAppsSubmitted()); assertEquals(appsPending, qm.getAppsPending()); assertEquals(appsRunning, qm.getAppsRunning()); assertEquals(appsCompleted, qm.getAppsCompleted()); assertEquals(allocatedContainers, qm.getAllocatedContainers()); assertEquals(availableMB, qm.getAvailableMB()); assertEquals(availableVirtualCores, qm.getAvailableVirtualCores()); assertEquals(allocatedMB, qm.getAllocatedMB()); assertEquals(allocatedVirtualCores, qm.getAllocatedVirtualCores()); }
@Test (timeout = 50000) public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.setQueues("default", new String[] {"default"}); conf.setCapacity("default", 100); FifoScheduler fs = new FifoScheduler(); fs.init(conf); fs.start(); // mock rmContext to avoid NPE. RMContext context = mock(RMContext.class); fs.reinitialize(conf, null); fs.setRMContext(context); RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2"); RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2, "127.0.0.3"); fs.handle(new NodeAddedSchedulerEvent(n1)); fs.handle(new NodeAddedSchedulerEvent(n2)); fs.handle(new NodeUpdateSchedulerEvent(n1)); Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB()); // reconnect n1 with downgraded memory n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1, "127.0.0.2"); fs.handle(new NodeRemovedSchedulerEvent(n1)); fs.handle(new NodeAddedSchedulerEvent(n1)); fs.handle(new NodeUpdateSchedulerEvent(n1)); Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB()); fs.stop(); }
assertEquals(1024, scheduler.getRootQueueMetrics().getAllocatedMB()); assertEquals(2, scheduler.getRootQueueMetrics().getAllocatedVirtualCores()); assertEquals(512, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
assertEquals(1024, scheduler.getRootQueueMetrics().getAllocatedMB()); assertEquals(2, scheduler.getRootQueueMetrics().getAllocatedVirtualCores()); assertEquals(512, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());