private Set<String> getRemovedStaticQueues( AllocationConfiguration queueInfo) { if (queueInfo == null || allocConf == null) { return Collections.emptySet(); } Set<String> removedStaticQueues = new HashSet<>(); for (Set<String> queues : allocConf.getConfiguredQueues().values()) { removedStaticQueues.addAll(queues); } for (Set<String> queues : queueInfo.getConfiguredQueues().values()) { removedStaticQueues.removeAll(queues); } return removedStaticQueues; }
private void ensureQueueExistsAndIsCompatibleAndIsStatic( AllocationConfiguration queueConf, FSQueueType queueType) { for (String name : queueConf.getConfiguredQueues().get(queueType)) { Boolean removed = removeEmptyIncompatibleQueues(name, queueType).orElse(null); if (Boolean.FALSE.equals(removed)) { incompatibleQueuesPendingRemoval.add( new IncompatibleQueueRemovalTask(name, queueType)); } else { FSQueue queue = getQueue(name, true, queueType, false, null); if (queue != null) { queue.setDynamic(false); } } } }
/** * For the given child queue, set the max resources based on the * parent queue's default child resource settings. This method assumes that * the child queue is ad hoc and hence does not do any safety checks around * overwriting existing max resource settings. * * @param parent the parent queue * @param child the child queue * @param queueConf the {@link AllocationConfiguration} */ private void setChildResourceLimits(FSParentQueue parent, FSQueue child, AllocationConfiguration queueConf) { Map<FSQueueType, Set<String>> configuredQueues = queueConf.getConfiguredQueues(); // Ad hoc queues do not exist in the configured queues map if (!configuredQueues.get(FSQueueType.LEAF).contains(child.getName()) && !configuredQueues.get(FSQueueType.PARENT).contains(child.getName())) { // For ad hoc queues, set their max reource allocations based on // their parents' default child settings. ConfigurableResource maxChild = parent.getMaxChildQueueResource(); if (maxChild != null) { child.setMaxShare(maxChild); } } }
/** * After reloading the allocation config, the max resource settings for any * ad hoc queues will be missing. This method goes through the queue manager's * queue list and adds back the max resources settings for any ad hoc queues. * Note that the new max resource settings will be based on the new config. * The old settings are lost. */ private void applyChildDefaults() { Collection<FSQueue> queues = queueMgr.getQueues(); Set<String> configuredLeafQueues = allocConf.getConfiguredQueues().get(FSQueueType.LEAF); Set<String> configuredParentQueues = allocConf.getConfiguredQueues().get(FSQueueType.PARENT); for (FSQueue queue : queues) { // If the queue is ad hoc and not root, apply the child defaults if ((queue.getParent() != null) && !configuredLeafQueues.contains(queue.getName()) && !configuredParentQueues.contains(queue.getName())) { ConfigurableResource max = queue.getParent(). getMaxChildQueueResource(); if (max != null) { queue.setMaxShare(max); } } } }
@Test public void testParentTagWithChild() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\"parent\" type=\"parent\">"); out.println(" <queue name=\"child\">"); out.println(" </queue>"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; // Check whether queue 'parent' and 'child' are loaded successfully assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT) .contains("root.parent")); assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.parent.child")); }
public void updateAllocationConfiguration(AllocationConfiguration queueConf) { for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) { if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { getLeafQueue(name, true); for (String name : queueConf.getConfiguredQueues().get( FSQueueType.PARENT)) { if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) {
public void updateAllocationConfiguration(AllocationConfiguration queueConf) { for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) { if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { getLeafQueue(name, true); for (String name : queueConf.getConfiguredQueues().get( FSQueueType.PARENT)) { if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) {
assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass()); assertEquals(1, allocConf.getQueueMaxApps("root.queueA")); assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueA")); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueB")); assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass()); assertEquals(3, allocConf.getQueueMaxApps("root.queueB")); assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueB"));
assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass()); assertEquals(1, allocConf.getQueueMaxApps("root.queueA")); assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueA")); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueB")); assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass()); assertEquals(3, allocConf.getQueueMaxApps("root.queueB")); assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueB"));
assertTrue(allocConf.isReservable(reservableQueueName)); Map<FSQueueType, Set<String>> configuredQueues = allocConf.getConfiguredQueues(); assertTrue("reservable queue is expected be to a parent queue", configuredQueues.get(FSQueueType.PARENT).contains(reservableQueueName));
AllocationConfiguration queueConf = confHolder.allocConf; assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
AllocationConfiguration queueConf = confHolder.allocConf; assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); queueConf.getFairSharePreemptionThreshold("root.queueG.queueH"), 0.01); assertTrue(queueConf.getConfiguredQueues() .get(FSQueueType.PARENT) .contains("root.queueF")); assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT) .contains("root.queueG")); assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueG.queueH"));
AllocationConfiguration queueConf = confHolder.allocConf; assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
AllocationConfiguration queueConf = confHolder.allocConf; assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); queueConf.getFairSharePreemptionThreshold("root.queueG.queueH"), 0.01); assertTrue(queueConf.getConfiguredQueues() .get(FSQueueType.PARENT) .contains("root.queueF")); assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT) .contains("root.queueG")); assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueG.queueH"));