private Collection<TempQueuePerPartition> getMostUnderservedQueues( PriorityQueue<TempQueuePerPartition> orderedByNeed, TQComparator tqComparator) { ArrayList<TempQueuePerPartition> underserved = new ArrayList<>(); while (!orderedByNeed.isEmpty()) { TempQueuePerPartition q1 = orderedByNeed.remove(); underserved.add(q1); // Add underserved queues in order for later uses context.addPartitionToUnderServedQueues(q1.queueName, q1.partition); TempQueuePerPartition q2 = orderedByNeed.peek(); // q1's pct of guaranteed won't be larger than q2's. If it's less, then // return what has already been collected. Otherwise, q1's pct of // guaranteed == that of q2, so add q2 to underserved list during the // next pass. if (q2 == null || tqComparator.compare(q1, q2) < 0) { if (null != q2) { context.addPartitionToUnderServedQueues(q2.queueName, q2.partition); } return underserved; } } return underserved; } }
for (String partition : context.getAllPartitions()) { LinkedHashSet<String> queueNames = context .getUnderServedQueuesPerPartition(partition); TempQueuePerPartition tq = context.getQueueByPartition(queueName, partition); LeafQueue leafQueue = tq.leafQueue; .getMinimumThresholdForIntraQueuePreemption()) { continue; tq, selectedCandidates, totalPreemptedResourceAllowed, queueReassignableResource, context.getMaxAllowableLimitForIntraQueuePreemption());
public long getMaximumKillWaitTimeMs() { if (maximumKillWaitTime > 0) { return maximumKillWaitTime; } return preemptionContext.getDefaultMaximumKillWaitTimeout(); }
public void computeIdealAllocation(Resource clusterResource, Resource totalPreemptionAllowed) { for (String partition : context.getAllPartitions()) { TempQueuePerPartition tRoot = context.getQueueByPartition( CapacitySchedulerConfiguration.ROOT, partition); updatePreemptableExtras(tRoot); // compute the ideal distribution of resources among queues // updates cloned queues state accordingly tRoot.initializeRootIdealWithGuarangeed(); recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed); } // based on ideal allocation select containers to be preempted from each // calculate resource-to-obtain by partition for each leaf queues calculateResToObtainByPartitionForLeafQueues(context.getLeafQueueNames(), clusterResource); } }
for (String q1 : preemptionContext.getLeafQueueNames()) { for (String q2 : preemptionContext.getLeafQueueNames()) { TempQueuePerPartition tq1 = preemptionContext.getQueueByPartition(q1, RMNodeLabelsManager.NO_LABEL); TempQueuePerPartition tq2 = preemptionContext.getQueueByPartition(q2, RMNodeLabelsManager.NO_LABEL);
if (context.getQueueByPartition(queueName, RMNodeLabelsManager.NO_LABEL).preemptionDisabled) { if (LOG.isDebugEnabled()) { for (TempQueuePerPartition qT : context.getQueuePartitions(queueName)) { qT.getUsed(), Resources .multiply(qT.getGuaranteed(), 1.0 + context.getMaxIgnoreOverCapacity()))) { Resource.newInstance(0, 0))) { resToObtain = Resources.multiplyAndNormalizeUp(rc, qT.toBePreempted, context.getNaturalTerminationFactor(), Resource.newInstance(1, 1));
for (String partition : preemptionContext.getAllPartitions()) { LinkedHashSet<String> queueNames = preemptionContext .getUnderServedQueuesPerPartition(partition); LeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName, RMNodeLabelsManager.NO_LABEL).leafQueue;
String partition = context.getScheduler() .getSchedulerNode(c.getAllocatedNode()).getPartition(); TempQueuePerPartition tq = context.getQueueByPartition(app.getQueueName(), partition); TempUserPerPartition tmpUser = tq.getUsersPerPartition().get(app.getUser()); && context.getIntraQueuePreemptionOrderPolicy() .equals(IntraQueuePreemptionOrderPolicy.USERLIMIT_FIRST);
curCandidates, totalPreemptionAllowed, false); if (!preemptionContext.isObserveOnly()) { preemptionContext.getRMContext().getDispatcher().getEventHandler() .handle(new ContainerPreemptEvent(appId, c, SchedulerEventType.KILL_RESERVED_CONTAINER)); if (null != preemptionContext.getKillableContainers() && preemptionContext .getKillableContainers().contains(c.getContainerId())) { continue;
if (null != preemptionContext.getKillableContainers() && preemptionContext .getKillableContainers().contains(c.getContainerId())) { continue; if (ret && preemptionContext.getIntraQueuePreemptionOrderPolicy() .equals(IntraQueuePreemptionOrderPolicy.USERLIMIT_FIRST)) { Resources.subtractFrom(rollingUsedResourcePerUser,
context.getIntraQueuePreemptionOrderPolicy());
for (String leafQueue : preemptionContext.getLeafQueueNames()) { queueToPreemptableResourceByPartition.put(leafQueue, CapacitySchedulerPreemptionUtils
for (String queueName : preemptionContext.getLeafQueueNames()) { if (preemptionContext.getQueueByPartition(queueName, RMNodeLabelsManager.NO_LABEL).preemptionDisabled) { if (LOG.isDebugEnabled()) { LeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName, RMNodeLabelsManager.NO_LABEL).leafQueue;
tq.leafQueue.getOrderingPolicy(); if (queueOrderingPolicy instanceof FairOrderingPolicy && (context.getIntraQueuePreemptionOrderPolicy() == IntraQueuePreemptionOrderPolicy.USERLIMIT_FIRST)) { reverseComp = Collections.reverseOrder(
tq.leafQueue.getOrderingPolicy(); if (orderingPolicy instanceof FairOrderingPolicy && (context.getIntraQueuePreemptionOrderPolicy() == IntraQueuePreemptionOrderPolicy.USERLIMIT_FIRST)) { taComparator = new TAFairOrderingComparator(this.rc, clusterResource);
context.addPartitionToUnderServedQueues(q1.queueName, q1.partition);
private void calculateToBePreemptedResourcePerApp(Resource clusterResource, TreeSet<TempAppPerPartition> orderedApps, Resource preemptionLimit) { for (TempAppPerPartition tmpApp : orderedApps) { if (Resources.lessThanOrEqual(rc, clusterResource, preemptionLimit, Resources.none()) || Resources.lessThanOrEqual(rc, clusterResource, tmpApp.getUsed(), Resources.none())) { continue; } Resource preemtableFromApp = Resources.subtract(tmpApp.getUsed(), tmpApp.idealAssigned); Resources.subtractFromNonNegative(preemtableFromApp, tmpApp.selected); Resources.subtractFromNonNegative(preemtableFromApp, tmpApp.getAMUsed()); if (context.getIntraQueuePreemptionOrderPolicy() .equals(IntraQueuePreemptionOrderPolicy.USERLIMIT_FIRST)) { Resources.subtractFromNonNegative(preemtableFromApp, tmpApp.getFiCaSchedulerApp().getCSLeafQueue().getMinimumAllocation()); } // Calculate toBePreempted from apps as follows: // app.preemptable = min(max(app.used - app.selected - app.ideal, 0), // intra_q_preemptable) tmpApp.toBePreempted = Resources.min(rc, clusterResource, Resources .max(rc, clusterResource, preemtableFromApp, Resources.none()), Resources.clone(preemptionLimit)); preemptionLimit = Resources.subtractFromNonNegative(preemptionLimit, tmpApp.toBePreempted); } }