/** * This method is visible to allow sub-classes to override the behavior, * specifically for federation purposes we do not want to cap resources as it * is done here. * * @param rc the {@code ResourceCalculator} to be used * @param clusterResource the total cluster resources * @param offered the resources offered to this queue * @return the amount of resources accepted after considering max and * deducting assigned. */ protected Resource filterByMaxDeductAssigned(ResourceCalculator rc, Resource clusterResource, Resource offered) { if (null == children || children.isEmpty()) { Resource maxOfGuranteedAndUsedDeductAssigned = Resources.subtract( Resources.max(rc, clusterResource, getUsed(), getGuaranteed()), idealAssigned); maxOfGuranteedAndUsedDeductAssigned = Resources.max(rc, clusterResource, maxOfGuranteedAndUsedDeductAssigned, Resources.none()); offered = Resources.min(rc, clusterResource, offered, maxOfGuranteedAndUsedDeductAssigned); } return offered; }
private void logToCSV(List<String> leafQueueNames){ Collections.sort(leafQueueNames); String queueState = " QUEUESTATE: " + clock.getTime(); StringBuilder sb = new StringBuilder(); sb.append(queueState); for (String queueName : leafQueueNames) { TempQueuePerPartition tq = getQueueByPartition(queueName, RMNodeLabelsManager.NO_LABEL); sb.append(", "); tq.appendLogString(sb); } LOG.debug(sb.toString()); }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(" NAME: " + queueName).append(" CUR: ").append(current) .append(" PEN: ").append(pending).append(" RESERVED: ").append(reserved) .append(" GAR: ").append(getGuaranteed()).append(" NORM: ") .append(Arrays.toString(normalizedGuarantee)) .append(" IDEAL_ASSIGNED: ").append(idealAssigned) .append(" IDEAL_PREEMPT: ").append(toBePreempted) .append(" ACTUAL_PREEMPT: ").append(getActuallyToBePreempted()) .append(" UNTOUCHABLE: ").append(untouchableExtra) .append(" PREEMPTABLE: ").append(preemptableExtra).append("\n"); return sb.toString(); }
/** * Do we allow the demanding queue preempt resource from other queues? * A satisfied queue is not allowed to preempt resource from other queues. * @param demandingQueue * @return allowed/not */ private boolean isQueueSatisfied(String demandingQueue, String partition) { TempQueuePerPartition tq = preemptionContext.getQueueByPartition( demandingQueue, partition); if (null == tq) { return false; } Resource guaranteed = tq.getGuaranteed(); Resource usedDeductReservd = Resources.subtract(tq.getUsed(), tq.getReserved()); Resource markedToPreemptFromOtherQueue = toPreemptedFromOtherQueues.get( demandingQueue, partition); if (null == markedToPreemptFromOtherQueue) { markedToPreemptFromOtherQueue = Resources.none(); } // return Used - reserved + to-preempt-from-other-queue >= guaranteed boolean flag = Resources.greaterThanOrEqual(rc, clusterResource, Resources.add(usedDeductReservd, markedToPreemptFromOtherQueue), guaranteed); return flag; }
Resource maxIntraQueuePreemptable = Resources.multiply(tq.getGuaranteed(), maxAllowablePreemptLimit); if (Resources.greaterThan(rc, clusterResource, maxIntraQueuePreemptable, tq.getActuallyToBePreempted())) { Resources.subtractFrom(maxIntraQueuePreemptable, tq.getActuallyToBePreempted()); } else { maxIntraQueuePreemptable = Resource.newInstance(0, 0); tq.addAllApps(orderedApps); (TreeSet<TempAppPerPartition>) orderedApps, tq.getUsersPerPartition(), context.getIntraQueuePreemptionOrderPolicy()); for (TempAppPerPartition tmpApp : tq.getApps()) { LOG.debug(tmpApp);
q.getGuaranteed(), Resources.none())) { nonZeroGuarQueues.add(q); } else { for (TempQueuePerPartition t:queues) { if (Resources.greaterThan(rc, tot_guarant, t.getUsed(), t.idealAssigned)) { Resources.addTo(totPreemptionNeeded, Resources .subtract(t.getUsed(), t.idealAssigned)); t.assignPreemption(scalingFactor, rc, tot_guarant);
qT.getUsed(), Resources .multiply(qT.getGuaranteed(), 1.0 + context.getMaxIgnoreOverCapacity()))) { qT.setActuallyToBePreempted(Resources.clone(resToObtain)); } else { qT.setActuallyToBePreempted(Resources.none());
for (Iterator<TempQueuePerPartition> i = qAlloc.iterator(); i.hasNext(); ) { TempQueuePerPartition q = i.next(); Resource used = q.getUsed(); if (Resources.greaterThan(rc, totGuarant, used, q.getGuaranteed())) { initIdealAssigned = Resources.add( Resources.componentwiseMin(q.getGuaranteed(), q.getUsed()), q.untouchableExtra); } else{ Resource curPlusPend = Resources.add(q.getUsed(), q.pending); if (Resources.lessThan(rc, totGuarant, q.idealAssigned, curPlusPend)) { orderedByNeed.add(q); Resource wQidle = sub.offer(wQavail, rc, totGuarant, isReservedPreemptionCandidatesSelector, allowQueuesBalanceAfterAllQueuesSatisfied);
boolean allowQueueBalanceAfterAllSafisfied) { Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax( Resources.subtract(getMax(), idealAssigned), Resource.newInstance(0, 0)); .subtract(Resources.add(getUsed(), (considersReservedResource ? pending : pendingDeductReserved)), idealAssigned))); accepted = filterByMaxDeductAssigned(rc, clusterResource, accepted); accepted = acceptedByLocality(rc, accepted);
/** * This method is visible to allow sub-classes to ovverride the behavior, * specifically for federation purposes we need to initialize per-sub-cluster * roots as well as the global one. */ protected void initializeRootIdealWithGuarangeed() { idealAssigned = Resources.clone(getGuaranteed()); }
Resource queueReassignableResource = Resources.subtract(tq.getUsed(), tq.getActuallyToBePreempted());
tq.deductActuallyToBePreempted(context.getResourceCalculator(), tq.totalPartitionResource, res); Collection<TempAppPerPartition> tas = tq.getApps(); if (null == tas || tas.isEmpty()) { continue;
.getUsersPerPartition(); tq.addUserPerPartition(userName, tmpUser);
ret = new TempQueuePerPartition(queueName, current, preemptionDisabled, partitionToLookAt, killable, absCap, absMaxCap, partitionResource, reserved, curQueue, effMinRes, effMaxRes); subq.relativePriority = c.getPriority().getPriority(); ret.addChild(subq); subq.parent = ret;
void appendLogString(StringBuilder sb) { sb.append(queueName).append(", ").append(current.getMemorySize()) .append(", ").append(current.getVirtualCores()).append(", ") .append(pending.getMemorySize()).append(", ") .append(pending.getVirtualCores()).append(", ") .append(getGuaranteed().getMemorySize()).append(", ") .append(getGuaranteed().getVirtualCores()).append(", ") .append(idealAssigned.getMemorySize()).append(", ") .append(idealAssigned.getVirtualCores()).append(", ") .append(toBePreempted.getMemorySize()).append(", ") .append(toBePreempted.getVirtualCores()).append(", ") .append(getActuallyToBePreempted().getMemorySize()).append(", ") .append(getActuallyToBePreempted().getVirtualCores()); }
private double getIdealPctOfGuaranteed(TempQueuePerPartition q) { double pctOver = Integer.MAX_VALUE; if (q != null && Resources.greaterThan(rc, clusterRes, q.getGuaranteed(), Resources.none())) { pctOver = Resources.divide(rc, clusterRes, q.idealAssigned, q.getGuaranteed()); } return (pctOver); } }
public void assignPreemption(float scalingFactor, ResourceCalculator rc, Resource clusterResource) { Resource usedDeductKillable = Resources.subtract(getUsed(), killable); Resource totalResource = Resources.add(getUsed(), pending); // The minimum resource that we need to keep for a queue is: // max(idealAssigned, min(used + pending, guaranteed)). // // Doing this because when we calculate ideal allocation doesn't consider // reserved resource, ideal-allocation calculated could be less than // guaranteed and total. We should avoid preempt from a queue if it is // already // <= its guaranteed resource. Resource minimumQueueResource = Resources.max(rc, clusterResource, Resources.min(rc, clusterResource, totalResource, getGuaranteed()), idealAssigned); if (Resources.greaterThan(rc, clusterResource, usedDeductKillable, minimumQueueResource)) { toBePreempted = Resources.multiply( Resources.subtract(usedDeductKillable, minimumQueueResource), scalingFactor); } else { toBePreempted = Resources.none(); } }
Resources.addTo(activeCap, q.getGuaranteed()); ResourceInformation nResourceInformation = q.getGuaranteed() .getResourceInformation(i); ResourceInformation dResourceInformation = activeCap
preemptableExtra = Resources.none(); Resource extra = Resources.subtract(getUsed(), getGuaranteed()); if (Resources.lessThan(rc, totalPartitionResource, extra, Resources.none())) {
assertEquals(0, tempQueueAPartition.untouchableExtra.getMemorySize()); long extraForQueueA = tempQueueAPartition.getUsed().getMemorySize() - tempQueueAPartition .getGuaranteed().getMemorySize(); assertEquals(extraForQueueA, tempQueueAPartition.preemptableExtra.getMemorySize());