public UserMetricsInfo(final ResourceManager rm, final String user) { ResourceScheduler rs = rm.getResourceScheduler(); QueueMetrics metrics = rs.getRootQueueMetrics(); QueueMetrics userMetrics = metrics.getUserMetrics(user); this.userMetricsAvailable = false; if (userMetrics != null) { this.userMetricsAvailable = true; this.appsSubmitted = userMetrics.getAppsSubmitted(); this.appsCompleted = userMetrics.getAppsCompleted(); this.appsPending = userMetrics.getAppsPending(); this.appsRunning = userMetrics.getAppsRunning(); this.appsFailed = userMetrics.getAppsFailed(); this.appsKilled = userMetrics.getAppsKilled(); this.runningContainers = userMetrics.getAllocatedContainers(); this.pendingContainers = userMetrics.getPendingContainers(); this.reservedContainers = userMetrics.getReservedContainers(); this.reservedMB = userMetrics.getReservedMB(); this.pendingMB = userMetrics.getPendingMB(); this.allocatedMB = userMetrics.getAllocatedMB(); this.reservedVirtualCores = userMetrics.getReservedVirtualCores(); this.pendingVirtualCores = userMetrics.getPendingVirtualCores(); this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores(); } }
public ClusterMetricsInfo(final ResourceManager rm) { ResourceScheduler rs = rm.getResourceScheduler(); QueueMetrics metrics = rs.getRootQueueMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); this.appsSubmitted = metrics.getAppsSubmitted(); this.appsCompleted = metrics.getAppsCompleted(); this.appsPending = metrics.getAppsPending(); this.appsRunning = metrics.getAppsRunning(); this.appsFailed = metrics.getAppsFailed(); this.appsKilled = metrics.getAppsKilled(); this.reservedMB = metrics.getReservedMB(); this.availableMB = metrics.getAvailableMB(); this.allocatedMB = metrics.getAllocatedMB(); this.reservedVirtualCores = metrics.getReservedVirtualCores(); this.availableVirtualCores = metrics.getAvailableVirtualCores(); this.allocatedVirtualCores = metrics.getAllocatedVirtualCores(); this.containersAllocated = metrics.getAllocatedContainers(); this.containersPending = metrics.getPendingContainers(); this.containersReserved = metrics.getReservedContainers(); this.totalMB = availableMB + allocatedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; this.activeNodes = clusterMetrics.getNumActiveNMs(); this.lostNodes = clusterMetrics.getNumLostNMs(); this.unhealthyNodes = clusterMetrics.getUnhealthyNMs(); this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs(); this.rebootedNodes = clusterMetrics.getNumRebootedNMs(); this.totalNodes = activeNodes + lostNodes + decommissionedNodes + rebootedNodes + unhealthyNodes; }
public synchronized void move(Queue newQueue) { QueueMetrics oldMetrics = queue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); String user = getUser(); for (RMContainer liveContainer : liveContainers.values()) { Resource resource = liveContainer.getContainer().getResource(); oldMetrics.releaseResources(user, 1, resource); newMetrics.allocateResources(user, 1, resource, false); } for (Map<NodeId, RMContainer> map : reservedContainers.values()) { for (RMContainer reservedContainer : map.values()) { Resource resource = reservedContainer.getReservedResource(); oldMetrics.unreserveResource(user, resource); newMetrics.reserveResource(user, resource); } } appSchedulingInfo.move(newQueue); this.queue = newQueue; }
private void resetQueueMetrics(QueueMetrics qm) { appsSubmittedCarryOn = qm.getAppsSubmitted(); appsPendingCarryOn = qm.getAppsPending(); appsRunningCarryOn = qm.getAppsRunning(); appsCompletedCarryOn = qm.getAppsCompleted(); }
QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, conf); MetricsSource queueSource= queueSource(ms, queueName); AppSchedulingInfo app = mockApp(user); metrics.submitApp(user); MetricsSource userSource = userSource(ms, queueName, user); checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3)); metrics.runAppAttempt(app.getApplicationId(), user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true); checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2)); checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); metrics.finishAppAttempt( app.getApplicationId(), app.isPending(), app.getUser()); checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.finishApp(user, RMAppState.FINISHED); checkApps(queueSource, 1, 0, 0, 1, 0, 0, true); assertNull(userSource);
private void assertMetrics(QueueMetrics qm, int appsSubmitted, int appsPending, int appsRunning, int appsCompleted, int allocatedContainers, int availableMB, int availableVirtualCores, int allocatedMB, int allocatedVirtualCores) { assertEquals(appsSubmitted, qm.getAppsSubmitted()); assertEquals(appsPending, qm.getAppsPending()); assertEquals(appsRunning, qm.getAppsRunning()); assertEquals(appsCompleted, qm.getAppsCompleted()); assertEquals(allocatedContainers, qm.getAllocatedContainers()); assertEquals(availableMB, qm.getAvailableMB()); assertEquals(availableVirtualCores, qm.getAvailableVirtualCores()); assertEquals(allocatedMB, qm.getAllocatedMB()); assertEquals(allocatedVirtualCores, qm.getAllocatedVirtualCores()); }
metrics.getAvailableMB() + metrics.getAllocatedMB(); long totalVirtualCoresExpect = metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores(); assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), submittedApps); assertEquals("appsCompleted doesn't match", metrics.getAppsCompleted(), completedApps); assertEquals("reservedMB doesn't match", metrics.getReservedMB(), reservedMB); assertEquals("availableMB doesn't match", metrics.getAvailableMB(), availableMB); assertEquals("allocatedMB doesn't match", metrics.getAllocatedMB(), allocMB); assertEquals("reservedVirtualCores doesn't match", metrics.getReservedVirtualCores(), reservedVirtualCores); assertEquals("availableVirtualCores doesn't match", metrics.getAvailableVirtualCores(), availableVirtualCores); assertEquals("allocatedVirtualCores doesn't match", totalVirtualCoresExpect, allocVirtualCores);
private void verifyMetrics(QueueMetrics metrics, long availableMB, int availableVirtualCores, long allocatedMB, int allocatedVirtualCores, int allocatedContainers) { Assert.assertEquals(availableMB, metrics.getAvailableMB()); Assert.assertEquals(availableVirtualCores, metrics.getAvailableVirtualCores()); Assert.assertEquals(allocatedMB, metrics.getAllocatedMB()); Assert.assertEquals(allocatedVirtualCores, metrics.getAllocatedVirtualCores()); Assert.assertEquals(allocatedContainers, metrics.getAllocatedContainers()); }
private void checkQueueMetrics(QueueMetrics metrics, int activeApps, int runningApps, int allocMb, int allocVcores, int reservedMb, int reservedVcores, int pendingMb, int pendingVcores) { assertEquals(activeApps, metrics.getActiveApps()); assertEquals(runningApps, metrics.getAppsRunning()); assertEquals(allocMb, metrics.getAllocatedMB()); assertEquals(allocVcores, metrics.getAllocatedVirtualCores()); assertEquals(reservedMb, metrics.getReservedMB()); assertEquals(reservedVcores, metrics.getReservedVirtualCores()); assertEquals(pendingMb, metrics.getPendingMB()); assertEquals(pendingVcores, metrics.getPendingVirtualCores()); }
private void verifyQueueMetrics(QueueMetrics queue) { Assert.assertEquals(0, queue.getPendingMB()); Assert.assertEquals(0, queue.getActiveUsers()); Assert.assertEquals(0, queue.getActiveApps()); Assert.assertEquals(0, queue.getAppsPending()); Assert.assertEquals(0, queue.getAppsRunning()); Assert.assertEquals(0, queue.getAllocatedMB()); Assert.assertEquals(0, queue.getAllocatedVirtualCores()); }
QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, new Configuration()); MetricsSource queueSource = queueSource(ms, queueName); AppSchedulingInfo app = mockApp(user); metrics.submitApp(user); MetricsSource userSource = userSource(ms, queueName, user); checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.runAppAttempt(app.getApplicationId(), user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); metrics.finishAppAttempt( app.getApplicationId(), app.isPending(), app.getUser()); checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.runAppAttempt(app.getApplicationId(), user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); metrics.finishAppAttempt( app.getApplicationId(), app.isPending(), app.getUser()); checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.submitAppAttempt(user);
try { assertMetric("appsSubmitted", appsSubmitted, metrics.getAppsSubmitted()); assertMetric("appsPending", appsPending, metrics.getAppsPending()); assertMetric("containersPending", containersPending, metrics.getPendingContainers()); assertMetric("availableMB", availableMB, metrics.getAvailableMB()); assertMetric("activeApplications", activeApplications, metrics.getActiveApps());
/** * Compute the maximum resource AM can use. The value is the result of * multiplying FairShare and maxAMShare. If FairShare is zero, use * min(maxShare, available resource) instead to prevent zero value for * maximum AM resource since it forbids any job running in the queue. * * @return the maximum resource AM can use */ private Resource computeMaxAMResource() { Resource maxResource = Resources.clone(getFairShare()); if (maxResource.getMemorySize() == 0) { maxResource.setMemorySize( Math.min(scheduler.getRootQueueMetrics().getAvailableMB(), getMaxShare().getMemorySize())); } if (maxResource.getVirtualCores() == 0) { maxResource.setVirtualCores(Math.min( scheduler.getRootQueueMetrics().getAvailableVirtualCores(), getMaxShare().getVirtualCores())); } // Round up to allow AM to run when there is only one vcore on the cluster return Resources.multiplyAndRoundUp(maxResource, maxAMShare); }
.put(newContainerId, rmContainer); QueueMetrics queueA1M = queueA1.getMetrics(); queueA1M.incrPendingResources(rmContainer.getNodeLabelExpression(), "user1", 1, resource); queueA1M.allocateResources(rmContainer.getNodeLabelExpression(), "user1", resource); QueueMetrics queueBM = queueB.getMetrics(); Assert.assertEquals(0, queueA1M.getAllocatedMB()); Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores()); Assert.assertEquals(1024, queueB1M.getAllocatedMB()); Assert.assertEquals(1, queueB1M.getAllocatedVirtualCores()); RMAppAttemptState.FINISHED, false)); Assert.assertEquals(0, queueA1M.getAllocatedMB()); Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores()); Assert.assertEquals(0, queueB1M.getAllocatedMB()); Assert.assertEquals(0, queueB1M.getAllocatedVirtualCores());
public void allocateResources(String user, int containers, Resource res, boolean decrPending) { allocatedContainers.incr(containers); aggregateContainersAllocated.incr(containers); allocatedMB.incr(res.getMemory() * containers); allocatedVCores.incr(res.getVirtualCores() * containers); if (decrPending) { _decrPendingResources(containers, res); } QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.allocateResources(user, containers, res, decrPending); } if (parent != null) { parent.allocateResources(user, containers, res, decrPending); } }
void reinitialize(boolean initialize) throws Exception { ClusterMetrics.destroy(); QueueMetrics.clearQueueMetrics(); if (initialize) { resetDispatcher(); createAndInitActiveServices(); } }
metrics.runAppAttempt(applicationId, user); + " type=" + type); metrics.allocateResources(user, 1, request.getCapability(), true); metrics.incrNodeTypeAggregations(user, type); return resourceRequests;
public void finishApp(String user, RMAppState rmAppFinalState) { switch (rmAppFinalState) { case KILLED: appsKilled.incr(); break; case FAILED: appsFailed.incr(); break; default: appsCompleted.incr(); break; } QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.finishApp(user, rmAppFinalState); } if (parent != null) { parent.finishApp(user, rmAppFinalState); } }
public synchronized void recoverContainer(RMContainer rmContainer) { QueueMetrics metrics = queue.getMetrics(); if (pending) { // If there was any container to recover, the application was // running from scheduler's POV. pending = false; metrics.runAppAttempt(applicationId, user); } // Container is completed. Skip recovering resources. if (rmContainer.getState().equals(RMContainerState.COMPLETED)) { return; } metrics.allocateResources(user, 1, rmContainer.getAllocatedResource(), false); }
public void finishAppAttempt( ApplicationId appId, boolean isPending, String user) { runBuckets.remove(appId); if (isPending) { appsPending.decr(); } else { appsRunning.decr(); } QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.finishAppAttempt(appId, isPending, user); } if (parent != null) { parent.finishAppAttempt(appId, isPending, user); } }