/** * Process resource update on a node. */ private synchronized void updateNodeAndQueueResource(RMNode nm, ResourceOption resourceOption) { updateNodeResource(nm, resourceOption); root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); }
/** * Process resource update on a node. */ private synchronized void updateNodeAndQueueResource(RMNode nm, ResourceOption resourceOption) { updateNodeResource(nm, resourceOption); root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); }
@Override public synchronized void updateClusterResource(Resource clusterResource, ResourceLimits resourceLimits) { // Update all children for (CSQueue childQueue : childQueues) { // Get ResourceLimits of child queue before assign containers ResourceLimits childLimits = getResourceLimitsOfChild(childQueue, clusterResource, resourceLimits); childQueue.updateClusterResource(clusterResource, childLimits); } // Update metrics CSQueueUtils.updateQueueStatistics( resourceCalculator, this, parent, clusterResource, minimumAllocation); }
@Override public synchronized void updateClusterResource(Resource clusterResource, ResourceLimits resourceLimits) { // Update all children for (CSQueue childQueue : childQueues) { // Get ResourceLimits of child queue before assign containers ResourceLimits childLimits = getResourceLimitsOfChild(childQueue, clusterResource, resourceLimits); childQueue.updateClusterResource(clusterResource, childLimits); } // Update metrics CSQueueUtils.updateQueueStatistics( resourceCalculator, this, parent, clusterResource, minimumAllocation); }
/** * Process resource update on a node. */ private void updateNodeAndQueueResource(RMNode nm, ResourceOption resourceOption) { try { writeLock.lock(); updateNodeResource(nm, resourceOption); Resource clusterResource = getClusterResource(); getRootQueue().updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); } finally { writeLock.unlock(); } }
/** * Process node labels update. */ private void updateNodeLabelsAndQueueResource( NodeLabelsUpdateSchedulerEvent labelUpdateEvent) { try { writeLock.lock(); Set<String> updateLabels = new HashSet<String>(); for (Entry<NodeId, Set<String>> entry : labelUpdateEvent .getUpdatedNodeToLabels().entrySet()) { NodeId id = entry.getKey(); Set<String> labels = entry.getValue(); FiCaSchedulerNode node = nodeTracker.getNode(id); if (node != null) { // Update old partition to list. updateLabels.add(node.getPartition()); } updateLabelsOnNode(id, labels); updateLabels.addAll(labels); } refreshLabelToNodeCache(updateLabels); Resource clusterResource = getClusterResource(); getRootQueue().updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); } finally { writeLock.unlock(); } }
@Override public void updateClusterResource(Resource clusterResource, ResourceLimits resourceLimits) { try { writeLock.lock(); // Update effective capacity in all parent queue. Set<String> configuredNodelabels = csContext.getConfiguration() .getConfiguredNodeLabels(getQueuePath()); for (String label : configuredNodelabels) { calculateEffectiveResourcesAndCapacity(label, clusterResource); } // Update all children for (CSQueue childQueue : childQueues) { // Get ResourceLimits of child queue before assign containers ResourceLimits childLimits = getResourceLimitsOfChild(childQueue, clusterResource, resourceLimits.getLimit(), RMNodeLabelsManager.NO_LABEL); childQueue.updateClusterResource(clusterResource, childLimits); } CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, this, labelManager, null); } finally { writeLock.unlock(); } }
private synchronized void addNode(RMNode nodeManager) { FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager, usePortForNodeName, nodeManager.getNodeLabels()); this.nodes.put(nodeManager.getNodeID(), schedulerNode); Resources.addTo(clusterResource, schedulerNode.getTotalResource()); // update this node to node label manager if (labelManager != null) { labelManager.activateNode(nodeManager.getNodeID(), schedulerNode.getTotalResource()); } root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); int numNodes = numNodeManagers.incrementAndGet(); updateMaximumAllocation(schedulerNode, true); LOG.info("Added node " + nodeManager.getNodeAddress() + " clusterResource: " + clusterResource); if (scheduleAsynchronously && numNodes == 1) { asyncSchedulerThread.beginSchedule(); } }
private synchronized void addNode(RMNode nodeManager) { FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager, usePortForNodeName, nodeManager.getNodeLabels()); this.nodes.put(nodeManager.getNodeID(), schedulerNode); Resources.addTo(clusterResource, schedulerNode.getTotalResource()); // update this node to node label manager if (labelManager != null) { labelManager.activateNode(nodeManager.getNodeID(), schedulerNode.getTotalResource()); } root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); int numNodes = numNodeManagers.incrementAndGet(); updateMaximumAllocation(schedulerNode, true); LOG.info("Added node " + nodeManager.getNodeAddress() + " clusterResource: " + clusterResource); if (scheduleAsynchronously && numNodes == 1) { asyncSchedulerThread.beginSchedule(); } }
@Lock(CapacityScheduler.class) private void reinitializeQueues(CapacitySchedulerConfiguration conf) throws IOException { // Parse new queues Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>(); CSQueue newRoot = parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, noop); // Ensure all existing queues are still present validateExistingQueues(queues, newQueues); // Add new queues addNewQueues(queues, newQueues); // Re-configure queues root.reinitialize(newRoot, clusterResource); initializeQueueMappings(); // Re-calculate headroom for active applications root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); labelManager.reinitializeQueueLabels(getQueueToLabels()); setQueueAcls(authorizer, queues); }
@Lock(CapacityScheduler.class) private void reinitializeQueues(CapacitySchedulerConfiguration conf) throws IOException { // Parse new queues Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>(); CSQueue newRoot = parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, noop); // Ensure all existing queues are still present validateExistingQueues(queues, newQueues); // Add new queues addNewQueues(queues, newQueues); // Re-configure queues root.reinitialize(newRoot, clusterResource); initializeQueueMappings(); // Re-calculate headroom for active applications root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); labelManager.reinitializeQueueLabels(getQueueToLabels()); setQueueAcls(authorizer, queues); }
queues, queues, TestUtils.spyHook); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); int numNodes = numNodeManagers.decrementAndGet();
getRootQueue().updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
@Override public void reinitializeQueues(CapacitySchedulerConfiguration newConf) throws IOException { // Parse new queues Map<String, CSQueue> newQueues = new HashMap<>(); CSQueue newRoot = parseQueue(this.csContext, newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP); // When failing over, if using configuration store, don't validate queue // hierarchy since queues can be removed without being STOPPED. if (!csContext.isConfigurationMutable() || csContext.getRMContext().getHAServiceState() != HAServiceProtocol.HAServiceState.STANDBY) { // Ensure queue hierarchy in the new XML file is proper. validateQueueHierarchy(queues, newQueues); } // Add new queues and delete OldQeueus only after validation. updateQueues(queues, newQueues); // Re-configure queues root.reinitialize(newRoot, this.csContext.getClusterResource()); setQueueAcls(authorizer, appPriorityACLManager, queues); // Re-calculate headroom for active applications Resource clusterResource = this.csContext.getClusterResource(); root.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); labelManager.reinitializeQueueLabels(getQueueToLabels()); this.queueStateManager.initialize(this); }
root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
Resources.createResource(numNodes * (80 * GB), numNodes * 100); when(csContext.getNumClusterNodes()).thenReturn(numNodes); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); assertEquals(Resource.newInstance(1 * GB, 1),