queuePath = q.getQueuePath(); capacity = q.getCapacity() * 100; usedCapacity = q.getUsedCapacity() * 100; maxCapacity = q.getMaximumCapacity(); if (maxCapacity < EPSILON || maxCapacity > 1f) maxCapacity = 1f; cap(q.getAbsoluteCapacity(), 0f, 1f) * 100; absoluteMaxCapacity = cap(q.getAbsoluteMaximumCapacity(), 0f, 1f) * 100; absoluteUsedCapacity = cap(q.getAbsoluteUsedCapacity(), 0f, 1f) * 100; numApplications = q.getNumApplications(); allocatedContainers = q.getMetrics().getAllocatedContainers(); pendingContainers = q.getMetrics().getPendingContainers(); reservedContainers = q.getMetrics().getReservedContainers(); queueName = q.getQueueName(); state = q.getState(); resourcesUsed = new ResourceInfo(q.getUsedResources()); if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) { hideReservationQueues = true; Set<String> labelSet = q.getAccessibleNodeLabels(); if (labelSet != null) { nodeLabels.addAll(labelSet); Collections.sort(nodeLabels); QueueCapacities qCapacities = q.getQueueCapacities();
CSQueue findQueue(CSQueue root, String queuePath) { if (root.getQueuePath().equals(queuePath)) { return root; } List<CSQueue> childQueues = root.getChildQueues(); if (childQueues != null) { for (CSQueue q : childQueues) { if (queuePath.startsWith(q.getQueuePath())) { CSQueue result = findQueue(q, queuePath); if (result != null) { return result; } } } } return null; }
csConf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS); CSQueue parentQ = q.getParent(); return csConf.getPreemptionDisabled(q.getQueuePath(), false); return csConf.getPreemptionDisabled(q.getQueuePath(), parentQ.getPreemptionDisabled());
private Map<String, Set<String>> getQueueToLabels() { Map<String, Set<String>> queueToLabels = new HashMap<String, Set<String>>(); for (CSQueue queue : queues.values()) { queueToLabels.put(queue.getQueueName(), queue.getAccessibleNodeLabels()); } return queueToLabels; }
String getChildQueuesToPrint() { StringBuilder sb = new StringBuilder(); for (CSQueue q : childQueues) { sb.append(q.getQueuePath() + "usedCapacity=(" + q.getUsedCapacity() + "), " + " label=(" + StringUtils.join(q.getAccessibleNodeLabels().iterator(), ",") + ")"); } return sb.toString(); }
public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { this.queueName = parent.getQueueName(); this.usedCapacity = parent.getUsedCapacity() * 100; this.capacity = parent.getCapacity() * 100; float max = parent.getMaximumCapacity(); if (max < EPSILON || max > 1f) max = 1f; this.maxCapacity = max * 100; capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), parent.getQueueResourceQuotas(), false); queues = getQueues(parent); health = new CapacitySchedulerHealthInfo(cs); }
CapacitySchedulerQueueInfo(final CSQueue q, final String nodeLabel) { QueueCapacities qCapacities = q.getQueueCapacities(); ResourceUsage queueResourceUsage = q.getQueueResourceUsage(); queuePath = q.getQueuePath(); capacity = qCapacities.getCapacity(nodeLabel) * 100; usedCapacity = q.getUsedCapacity(nodeLabel) * 100; maxCapacity = qCapacities.getMaximumCapacity(nodeLabel); if (maxCapacity < EPSILON || maxCapacity > 1f) maxCapacity = 1f; maxCapacity *= 100; absoluteCapacity = cap(qCapacities.getAbsoluteCapacity(nodeLabel), 0f, 1f) * 100; absoluteMaxCapacity = cap(qCapacities.getAbsoluteMaximumCapacity(nodeLabel), 0f, 1f) * 100; absoluteUsedCapacity = q.getAbsoluteUsedCapacity(nodeLabel) * 100; numApplications = q.getNumApplications(); queueName = q.getQueueName(); state = q.getState(); resourcesUsed = new ResourceInfo(queueResourceUsage.getUsed(nodeLabel)); if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) { hideReservationQueues = true; } // add labels Set<String> labelSet = q.getAccessibleNodeLabels(); if (labelSet != null) { nodeLabels.addAll(labelSet); Collections.sort(nodeLabels); } }
resUsagePerQueue.setUsed(used[i]); resUsagePerQueue.setReserved(reserved[i]); when(q.getQueueResourceUsage()).thenReturn(resUsagePerQueue); } else { q = mockLeafQueue(p, tot, i, abs, used, pending, reserved, apps, gran); when(q.getParent()).thenReturn(p); when(q.getQueueName()).thenReturn(queueName); when(q.getAbsoluteUsedCapacity()).thenReturn( Resources.divide(rc, tot, used[i], tot)); when(q.getAbsoluteCapacity()).thenReturn( Resources.divide(rc, tot, abs[i], tot)); when(q.getAbsoluteMaximumCapacity()).thenReturn( maxCap[i] / (float) tot.getMemorySize()); qc.setAbsoluteCapacity(Resources.divide(rc, tot, abs[i], tot)); qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize()); when(q.getQueueCapacities()).thenReturn(qc); Resource.newInstance(maxCap[i], 0)); qr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[i]); when(q.getQueueResourceQuotas()).thenReturn(qr); when(q.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL)) .thenReturn(abs[i]); when(q.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL)) .thenReturn(Resource.newInstance(maxCap[i], 0)); String queuePathName = (parentPathName + "." + queueName).replace("/", "root");
ResourceUsage resUsagePerQueue = new ResourceUsage(); resUsagePerQueue.setUsed(used[i]); when(q.getQueueResourceUsage()).thenReturn(resUsagePerQueue); } else { q = mockLeafQueue(p, tot, i, abs, used, pending, reserved, apps, gran); when(q.getParent()).thenReturn(p); when(q.getQueueName()).thenReturn(queueName); when(q.getAbsoluteUsedCapacity()).thenReturn( Resources.divide(rc, tot, used[i], tot)); when(q.getAbsoluteCapacity()).thenReturn( Resources.divide(rc, tot, abs[i], tot)); when(q.getAbsoluteMaximumCapacity()).thenReturn( maxCap[i] / (float) tot.getMemory()); String parentPathName = p.getQueuePath(); parentPathName = (parentPathName == null) ? "root" : parentPathName; String queuePathName = (parentPathName+"."+queueName).replace("/","root"); when(q.getQueuePath()).thenReturn(queuePathName); preemptionDisabled = mockPreemptionStatus(queuePathName); when(q.getPreemptionDisabled()).thenReturn(preemptionDisabled);
TempQueue ret; synchronized (root) { String queueName = root.getQueueName(); float absCap = root.getAbsoluteCapacity(); float absMaxCap = root.getAbsoluteMaximumCapacity(); boolean preemptionDisabled = root.getPreemptionDisabled(); Resource current = root.getQueueResourceUsage().getUsed(); Resource guaranteed = Resources.multiply(clusterResources, absCap); Resource maxCapacity = Resources.multiply(clusterResources, absMaxCap); } else { Resource pending = Resource.newInstance(0, 0); ret = new TempQueue(root.getQueueName(), current, pending, guaranteed, maxCapacity, false); Resource childrensPreemptable = Resource.newInstance(0, 0); for (CSQueue c : root.getChildQueues()) { TempQueue subq = cloneQueues(c, clusterResources); Resources.addTo(childrensPreemptable, subq.preemptableExtra);
Resource partitionResource, String partitionToLookAt) { TempQueuePerPartition ret; ReadLock readLock = curQueue.getReadLock(); try { String queueName = curQueue.getQueueName(); QueueCapacities qc = curQueue.getQueueCapacities(); float absCap = qc.getAbsoluteCapacity(partitionToLookAt); float absMaxCap = qc.getAbsoluteMaximumCapacity(partitionToLookAt); boolean preemptionDisabled = curQueue.getPreemptionDisabled(); .getQueueResourceQuotas(); Resource effMinRes = queueResourceQuotas .getEffectiveMinResource(partitionToLookAt); .clone(curQueue.getQueueResourceUsage().getUsed(partitionToLookAt)); Resource killable = Resources.none(); curQueue.getQueueResourceUsage().getReserved(partitionToLookAt)); if (null != preemptableQueues.get(queueName)) { killable = Resources.clone(preemptableQueues.get(queueName) for (CSQueue c : curQueue.getChildQueues()) { TempQueuePerPartition subq = cloneQueues(c, partitionResource, partitionToLookAt); CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY, configuredOrderingPolicy)) { subq.relativePriority = c.getPriority().getPriority();
private void checkQueueCapacity(CSQueue q, float expectedCapacity, float expectedAbsCapacity, float expectedMaxCapacity, float expectedAbsMaxCapacity) { final float epsilon = 1e-5f; assertEquals("capacity", expectedCapacity, q.getCapacity(), epsilon); assertEquals("absolute capacity", expectedAbsCapacity, q.getAbsoluteCapacity(), epsilon); assertEquals("maximum capacity", expectedMaxCapacity, q.getMaximumCapacity(), epsilon); assertEquals("absolute maximum capacity", expectedAbsMaxCapacity, q.getAbsoluteMaximumCapacity(), epsilon); }
numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); queues.get(CapacitySchedulerConfiguration.ROOT).getQueueResourceUsage() .incPending(Resources.createResource(1 * GB)); a.getQueueResourceUsage().incPending(Resources.createResource(1 * GB)); b.getQueueResourceUsage().incPending(Resources.createResource(1 * GB)); c.getQueueResourceUsage().incPending(Resources.createResource(1 * GB)); d.getQueueResourceUsage().incPending(Resources.createResource(1 * GB)); stubQueueAllocation(c, clusterResource, node_0, 0*GB); stubQueueAllocation(d, clusterResource, node_0, 0*GB); root.assignContainers(clusterResource, node_0, new ResourceLimits( clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); for(int i=0; i < 2; i++) stubQueueAllocation(c, clusterResource, node_0, 0*GB); stubQueueAllocation(d, clusterResource, node_0, 0*GB); root.assignContainers(clusterResource, node_0, new ResourceLimits( clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); stubQueueAllocation(c, clusterResource, node_0, 1*GB); stubQueueAllocation(d, clusterResource, node_0, 0*GB); root.assignContainers(clusterResource, node_0, new ResourceLimits( clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); stubQueueAllocation(c, clusterResource, node_0, 0*GB);
private void verifyQueueMetrics(CSQueue queue, int expectedMemory, Resource clusterResource) { assertEquals( computeQueueAbsoluteUsedCapacity(queue, expectedMemory, clusterResource), queue.getAbsoluteUsedCapacity(), DELTA); assertEquals( computeQueueUsedCapacity(queue, expectedMemory, clusterResource), queue.getUsedCapacity(), DELTA); }
Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); Assert.assertEquals(0.15, a.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals(0.70, c.getAbsoluteCapacity(), DELTA); Assert.assertEquals(0.70, c.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals(0.10 * 0.6, a1.getAbsoluteCapacity(), DELTA); Assert.assertEquals(0.15, a1.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals(0.10 * 0.4, a2.getAbsoluteCapacity(), DELTA); Assert.assertEquals(0.15, a2.getAbsoluteMaximumCapacity(), DELTA);
throw new Exception(); } catch (Exception e) { LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() + " alloc=" + allocation + " node=" + node.getNodeName()); .assignContainers(eq(clusterResource), any(CandidateNodeSet.class), any(ResourceLimits.class), any(SchedulingMode.class));
void printString(CSQueue nq, String indent) { if (nq instanceof ParentQueue) { System.out.println(indent + nq.getQueueName() + " cur:" + nq.getAbsoluteUsedCapacity() + " guar:" + nq.getAbsoluteCapacity() ); for (CSQueue q : ((ParentQueue)nq).getChildQueues()) { printString(q, indent + " "); } } else { System.out.println(indent + nq.getQueueName() + " pen:" + ((LeafQueue) nq) .getTotalPendingResourcesConsideringUserLimit(isA(Resource.class)) + " cur:" + nq.getAbsoluteUsedCapacity() + " guar:" + nq.getAbsoluteCapacity() ); for (FiCaSchedulerApp a : ((LeafQueue)nq).getApplications()) { System.out.println(indent + " " + a.getApplicationId()); } } }
protected float sumOfChildAbsCapacities() { try { writeLock.lock(); float ret = 0; for (CSQueue l : childQueues) { ret += l.getAbsoluteCapacity(); } return ret; } finally { writeLock.unlock(); } }
private int getNumAppsInQueue(String name, List<CSQueue> queues) { for (CSQueue queue : queues) { if (queue.getQueueName().equals(name)) { return queue.getNumApplications(); } } return -1; }
CSQueue childQueue = iter.next(); if(LOG.isDebugEnabled()) { LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath() + " stats: " + childQueue); getResourceLimitsOfChild(childQueue, cluster, limits); assignment = childQueue.assignContainers(cluster, node, childLimits); if(LOG.isDebugEnabled()) { LOG.debug("Assigned to queue: " + childQueue.getQueuePath() + " stats: " + childQueue + " --> " + assignment.getResource() + ", " + assignment.getType()); LOG.info("Re-sorting assigned queue: " + childQueue.getQueuePath() + " stats: " + childQueue); childQueues.add(childQueue);