private FSQueue getQueue(String name, boolean create, FSQueueType queueType) { name = ensureRootPrefix(name); synchronized (queues) { FSQueue queue = queues.get(name); if (queue == null && create) { // if the queue doesn't exist,create it and return queue = createQueue(name, queueType); // Update steady fair share for all queues if (queue != null) { rootQueue.recomputeSteadyShares(); } } return queue; } }
private FSQueue getQueue(String name, boolean create, FSQueueType queueType) { name = ensureRootPrefix(name); synchronized (queues) { FSQueue queue = queues.get(name); if (queue == null && create) { // if the queue doesn't exist,create it and return queue = createQueue(name, queueType); // Update steady fair share for all queues if (queue != null) { rootQueue.recomputeSteadyShares(); } } return queue; } }
void recomputeSteadyShares() { readLock.lock(); try { policy.computeSteadyShares(childQueues, getSteadyFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics() .setSteadyFairShare(childQueue.getSteadyFairShare()); if (childQueue instanceof FSParentQueue) { ((FSParentQueue) childQueue).recomputeSteadyShares(); } } } finally { readLock.unlock(); } }
private FSQueue getQueue(String name, boolean create, FSQueueType queueType, boolean recomputeSteadyShares, ApplicationId applicationId) { boolean recompute = recomputeSteadyShares; name = ensureRootPrefix(name); FSQueue queue; synchronized (queues) { queue = queues.get(name); if (queue == null && create) { // if the queue doesn't exist,create it and return queue = createQueue(name, queueType); } else { recompute = false; } // At this point the queue exists and we need to assign the app if to the // but only to a leaf queue if (applicationId != null && queue instanceof FSLeafQueue) { ((FSLeafQueue)queue).addAssignedApp(applicationId); } } // Don't recompute if it is an existing queue or no change was made if (recompute && queue != null) { rootQueue.recomputeSteadyShares(); } return queue; }
/** * Process resource update on a node and update Queue. */ @Override public void updateNodeResource(RMNode nm, ResourceOption resourceOption) { writeLock.lock(); try { super.updateNodeResource(nm, resourceOption); updateRootQueueMetrics(); queueMgr.getRootQueue().setSteadyFairShare(getClusterResource()); queueMgr.getRootQueue().recomputeSteadyShares(); } finally { writeLock.unlock(); } }
public void updateAllocationConfiguration(AllocationConfiguration queueConf) { // Create leaf queues and the parent queues in a leaf's // ancestry if they do not exist synchronized (queues) { // Verify and set scheduling policies for existing queues before creating // any queue, since we need parent policies to determine if we can create // its children. if (!rootQueue.verifyAndSetPolicyFromConf(queueConf)) { LOG.error("Setting scheduling policies for existing queues failed!"); } ensureQueueExistsAndIsCompatibleAndIsStatic(queueConf, FSQueueType.LEAF); // At this point all leaves and 'parents with // at least one child' would have been created. // Now create parents with no configured leaf. ensureQueueExistsAndIsCompatibleAndIsStatic(queueConf, FSQueueType.PARENT); } // Initialize all queues recursively rootQueue.reinit(true); // Update steady fair shares for all queues rootQueue.recomputeSteadyShares(); }
public void recomputeSteadyShares() { policy.computeSteadyShares(childQueues, getSteadyFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics().setSteadyFairShare(childQueue.getSteadyFairShare()); if (childQueue instanceof FSParentQueue) { ((FSParentQueue) childQueue).recomputeSteadyShares(); } } }
/** * Process resource update on a node and update Queue. */ @Override public synchronized void updateNodeResource(RMNode nm, ResourceOption resourceOption) { super.updateNodeResource(nm, resourceOption); updateRootQueueMetrics(); queueMgr.getRootQueue().setSteadyFairShare(clusterResource); queueMgr.getRootQueue().recomputeSteadyShares(); }
/** * Process resource update on a node and update Queue. */ @Override public synchronized void updateNodeResource(RMNode nm, ResourceOption resourceOption) { super.updateNodeResource(nm, resourceOption); updateRootQueueMetrics(); queueMgr.getRootQueue().setSteadyFairShare(clusterResource); queueMgr.getRootQueue().recomputeSteadyShares(); }
public void recomputeSteadyShares() { policy.computeSteadyShares(childQueues, getSteadyFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics().setSteadyFairShare(childQueue.getSteadyFairShare()); if (childQueue instanceof FSParentQueue) { ((FSParentQueue) childQueue).recomputeSteadyShares(); } } }
private synchronized void addNode(RMNode node) { FSSchedulerNode schedulerNode = new FSSchedulerNode(node, usePortForNodeName); nodes.put(node.getNodeID(), schedulerNode); Resources.addTo(clusterResource, schedulerNode.getTotalResource()); updateRootQueueMetrics(); updateMaximumAllocation(schedulerNode, true); queueMgr.getRootQueue().setSteadyFairShare(clusterResource); queueMgr.getRootQueue().recomputeSteadyShares(); LOG.info("Added node " + node.getNodeAddress() + " cluster capacity: " + clusterResource); }
private synchronized void addNode(RMNode node) { FSSchedulerNode schedulerNode = new FSSchedulerNode(node, usePortForNodeName); nodes.put(node.getNodeID(), schedulerNode); Resources.addTo(clusterResource, schedulerNode.getTotalResource()); updateRootQueueMetrics(); updateMaximumAllocation(schedulerNode, true); queueMgr.getRootQueue().setSteadyFairShare(clusterResource); queueMgr.getRootQueue().recomputeSteadyShares(); LOG.info("Added node " + node.getNodeAddress() + " cluster capacity: " + clusterResource); }
private void addNode(List<NMContainerStatus> containerReports, RMNode node) { writeLock.lock(); try { FSSchedulerNode schedulerNode = new FSSchedulerNode(node, usePortForNodeName); nodeTracker.addNode(schedulerNode); triggerUpdate(); Resource clusterResource = getClusterResource(); queueMgr.getRootQueue().setSteadyFairShare(clusterResource); queueMgr.getRootQueue().recomputeSteadyShares(); LOG.info("Added node " + node.getNodeAddress() + " cluster capacity: " + clusterResource); recoverContainersOnNode(containerReports, node); updateRootQueueMetrics(); } finally { writeLock.unlock(); } }
Resource clusterResource = getClusterResource(); queueMgr.getRootQueue().setSteadyFairShare(clusterResource); queueMgr.getRootQueue().recomputeSteadyShares(); updateRootQueueMetrics(); triggerUpdate();
queueMgr.getRootQueue().recomputeSteadyShares(); updateMaximumAllocation(node, false); LOG.info("Removed node " + rmNode.getNodeAddress() +
queueMgr.getRootQueue().recomputeSteadyShares(); updateMaximumAllocation(node, false); LOG.info("Removed node " + rmNode.getNodeAddress() +
@Test public void testSimpleFairShareCalculation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Have two queues which want entire cluster capacity createSchedulingRequest(10 * 1024, "queue1", "user1"); createSchedulingRequest(10 * 1024, "queue2", "user1"); createSchedulingRequest(10 * 1024, "root.default", "user1"); scheduler.update(); scheduler.getQueueManager().getRootQueue() .setSteadyFairShare(scheduler.getClusterResource()); scheduler.getQueueManager().getRootQueue().recomputeSteadyShares(); Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); // Divided three ways - between the two queues and the default queue for (FSLeafQueue p : queues) { assertEquals(3414, p.getFairShare().getMemorySize()); assertEquals(3414, p.getMetrics().getFairShareMB()); assertEquals(3414, p.getSteadyFairShare().getMemorySize()); assertEquals(3414, p.getMetrics().getSteadyFairShareMB()); } }
@Test public void testSimpleFairShareCalculation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Have two queues which want entire cluster capacity createSchedulingRequest(10 * 1024, "queue1", "user1"); createSchedulingRequest(10 * 1024, "queue2", "user1"); createSchedulingRequest(10 * 1024, "root.default", "user1"); scheduler.update(); scheduler.getQueueManager().getRootQueue() .setSteadyFairShare(scheduler.getClusterResource()); scheduler.getQueueManager().getRootQueue().recomputeSteadyShares(); Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); // Divided three ways - between the two queues and the default queue for (FSLeafQueue p : queues) { assertEquals(3414, p.getFairShare().getMemory()); assertEquals(3414, p.getMetrics().getFairShareMB()); assertEquals(3414, p.getSteadyFairShare().getMemory()); assertEquals(3414, p.getMetrics().getSteadyFairShareMB()); } }
scheduler.getQueueManager().getRootQueue() .setSteadyFairShare(scheduler.getClusterResource()); scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();