+ queueConf.getDefaultSchedulingPolicy() + " on new leaf queue.", ex); parent.addChildQueue(leafQueue); queues.put(leafQueue.getName(), leafQueue); leafQueues.add(leafQueue); return leafQueue; } else { FSParentQueue newParent = new FSParentQueue(queueName, scheduler, parent); try { newParent.setPolicy(queueConf.getDefaultSchedulingPolicy()); } catch (AllocationConfigurationException ex) { LOG.warn("Failed to set default scheduling policy " + queueConf.getDefaultSchedulingPolicy() + " on new parent queue.", ex); parent.addChildQueue(newParent); queues.put(newParent.getName(), newParent); newParent.updatePreemptionVariables(); parent = newParent;
@Override protected void dumpStateInternal(StringBuilder sb) { sb.append("{Name: " + getName() + ", Weight: " + weights + ", Policy: " + policy.getName() + ", FairShare: " + getFairShare() + ", SteadyFairShare: " + getSteadyFairShare() + ", MaxShare: " + getMaxShare() + ", MinShare: " + minShare + ", ResourceUsage: " + getResourceUsage() + ", Demand: " + getDemand() + ", MaxAMShare: " + maxAMShare + ", Runnable: " + getNumRunnableApps() + "}"); for(FSQueue child : getChildQueues()) { sb.append(", "); child.dumpStateInternal(sb); } } }
if (!parent.getPolicy().isChildPolicyAllowed(childPolicy)) { LOG.error("Can't create queue '" + queueName + "'," + "the child scheduling policy is not allowed by parent queue!"); return null; newParent = new FSParentQueue(queueName, scheduler, parent); queue = newParent; parent.addChildQueue(queue); setChildResourceLimits(parent, queue, queueConf); queues.put(queue.getName(), queue);
public void initialize(Configuration conf) throws IOException, SAXException, AllocationConfigurationException, ParserConfigurationException { rootQueue = new FSParentQueue("root", scheduler, null); queues.put(rootQueue.getName(), rootQueue); // Create the default queue getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true); }
@Override public Resource getMaximumContainerAllocation() { if (getName().equals("root")) { return maxContainerAllocation; } if (maxContainerAllocation.equals(Resources.unbounded()) && getParent() != null) { return getParent().getMaximumContainerAllocation(); } else { return maxContainerAllocation; } }
/** * Process resource update on a node and update Queue. */ @Override public synchronized void updateNodeResource(RMNode nm, ResourceOption resourceOption) { super.updateNodeResource(nm, resourceOption); updateRootQueueMetrics(); queueMgr.getRootQueue().setSteadyFairShare(clusterResource); queueMgr.getRootQueue().recomputeSteadyShares(); }
Thread.sleep(100); if (assumedFairShare.equals(((FairScheduler)rm.getResourceScheduler()) .getQueueManager().getRootQueue().getFairShare())) { break; FSParentQueue root = scheduler.getQueueManager().getRootQueue(); assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy); assertEquals(usedResources,root.getResourceUsage());
void recomputeSteadyShares() { readLock.lock(); try { policy.computeSteadyShares(childQueues, getSteadyFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics() .setSteadyFairShare(childQueue.getSteadyFairShare()); if (childQueue instanceof FSParentQueue) { ((FSParentQueue) childQueue).recomputeSteadyShares(); } } } finally { readLock.unlock(); } }
FSParentQueue parent = queue.getParent(); while (parent != null) { if (parent.getNumRunnableApps() == allocConf.getQueueMaxApps(parent .getName()) - 1) { highestQueueWithAppsNowRunnable = parent; parent = parent.getParent();
@Override protected Resource getPlanResources(Plan plan, Queue queue, Resource clusterResources) { FSParentQueue planQueue = (FSParentQueue)queue; Resource planResources = planQueue.getSteadyFairShare(); return planResources; }
public void initialize(Configuration conf) throws IOException, SAXException, AllocationConfigurationException, ParserConfigurationException { // Policies of root and default queue are set to // SchedulingPolicy.DEFAULT_POLICY since the allocation file hasn't been // loaded yet. rootQueue = new FSParentQueue("root", scheduler, null); rootQueue.setDynamic(false); queues.put(rootQueue.getName(), rootQueue); // Create the default queue FSLeafQueue defaultQueue = getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true); defaultQueue.setDynamic(false); // Recursively reinitialize to propagate queue properties rootQueue.reinit(true); }
@Test public void testMoveNonRunnableApp() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); assertEquals(0, oldQueue.getNumRunnableApps()); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); assertEquals(0, oldQueue.getNumRunnableApps()); assertEquals(0, targetQueue.getNumRunnableApps()); assertEquals(0, queueMgr.getRootQueue().getNumRunnableApps()); }
@Test public void testLowestCommonAncestorDeeperHierarchy() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); FSQueue aQueue = mock(FSLeafQueue.class); FSQueue bQueue = mock(FSLeafQueue.class); FSQueue a1Queue = mock(FSLeafQueue.class); FSQueue b1Queue = mock(FSLeafQueue.class); when(a1Queue.getName()).thenReturn("root.queue1.a.a1"); when(b1Queue.getName()).thenReturn("root.queue1.b.b1"); when(aQueue.getChildQueues()).thenReturn(Arrays.asList(a1Queue)); when(bQueue.getChildQueues()).thenReturn(Arrays.asList(b1Queue)); QueueManager queueManager = scheduler.getQueueManager(); FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(a1Queue, b1Queue); assertEquals(ancestorQueue, queue1); }
/** * Removes all empty dynamic queues (including empty dynamic parent queues). */ public void removeEmptyDynamicQueues() { synchronized (queues) { Set<FSParentQueue> parentQueuesToCheck = new HashSet<>(); for (FSQueue queue : getQueues()) { if (queue.isDynamic() && queue.getChildQueues().isEmpty()) { boolean removed = removeQueueIfEmpty(queue); if (removed && queue.getParent().isDynamic()) { parentQueuesToCheck.add(queue.getParent()); } } } while (!parentQueuesToCheck.isEmpty()) { FSParentQueue queue = parentQueuesToCheck.iterator().next(); if (queue.getChildQueues().isEmpty()) { removeQueue(queue); if (queue.getParent().isDynamic()) { parentQueuesToCheck.add(queue.getParent()); } } parentQueuesToCheck.remove(queue); } } }
/** * Remove a queue and all its descendents. */ private void removeQueue(FSQueue queue) { if (queue instanceof FSLeafQueue) { leafQueues.remove(queue); } else { List<FSQueue> childQueues = queue.getChildQueues(); while (!childQueues.isEmpty()) { removeQueue(childQueues.get(0)); } } queues.remove(queue.getName()); queue.getParent().getChildQueues().remove(queue); }
FSParentQueue parent = queue.getParent(); while (parent != null) { if (parent.getNumRunnableApps() == parent.getMaxRunningApps() - 1) { highestQueueWithAppsNowRunnable = parent; parent = parent.getParent();
rootQueue.recomputeSteadyShares(); rootQueue.updatePreemptionVariables();
bQueue.addApp(app2, true); queue1.setMaxShare(maxResourceConf); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); queue1.updateDemand(); Resources.equals(queue1.getDemand(), maxResource)); assertTrue("Demand of child queue not updated ", Resources.equals(aQueue.getDemand(), maxResource) &&
public void updateAllocationConfiguration(AllocationConfiguration queueConf) { // Create leaf queues and the parent queues in a leaf's // ancestry if they do not exist synchronized (queues) { // Verify and set scheduling policies for existing queues before creating // any queue, since we need parent policies to determine if we can create // its children. if (!rootQueue.verifyAndSetPolicyFromConf(queueConf)) { LOG.error("Setting scheduling policies for existing queues failed!"); } ensureQueueExistsAndIsCompatibleAndIsStatic(queueConf, FSQueueType.LEAF); // At this point all leaves and 'parents with // at least one child' would have been created. // Now create parents with no configured leaf. ensureQueueExistsAndIsCompatibleAndIsStatic(queueConf, FSQueueType.PARENT); } // Initialize all queues recursively rootQueue.reinit(true); // Update steady fair shares for all queues rootQueue.recomputeSteadyShares(); }
@Override void updateInternal() { readLock.lock(); try { policy.computeShares(childQueues, getFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics().setFairShare(childQueue.getFairShare()); childQueue.updateInternal(); } } finally { readLock.unlock(); } }