InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, ReservationAgent agent, Resource totalCapacity, long step, ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc, String queueName, Planner replanner, boolean getMoveOnExpiry, Clock clock) { this.queueMetrics = queueMetrics; this.policy = policy; this.agent = agent; this.step = step; this.totalCapacity = totalCapacity; this.resCalc = resCalc; this.minAlloc = minAlloc; this.maxAlloc = maxAlloc; this.rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc); this.queueName = queueName; this.replanner = replanner; this.getMoveOnExpiry = getMoveOnExpiry; this.clock = clock; }
InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, ReservationAgent agent, Resource totalCapacity, long step, ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc, String queueName, Planner replanner, boolean getMoveOnExpiry, Clock clock) { this.queueMetrics = queueMetrics; this.policy = policy; this.agent = agent; this.step = step; this.totalCapacity = totalCapacity; this.resCalc = resCalc; this.minAlloc = minAlloc; this.maxAlloc = maxAlloc; this.rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc); this.queueName = queueName; this.replanner = replanner; this.getMoveOnExpiry = getMoveOnExpiry; this.clock = clock; }
@Override public RLESparseResourceAllocation getReservationCountForUserOverTime( String user, long start, long end) { readLock.lock(); try { RLESparseResourceAllocation userResAlloc = userActiveReservationCount.get(user); if (userResAlloc != null) { return userResAlloc.getRangeOverlapping(start, end); } else { return new RLESparseResourceAllocation(resCalc); } } finally { readLock.unlock(); } }
private RLESparseResourceAllocation getUserRLEResourceAllocation(String user, long period) { RLESparseResourceAllocation resAlloc = null; if (period > 0) { if (userPeriodicResourceAlloc.containsKey(user)) { resAlloc = userPeriodicResourceAlloc.get(user); } else { resAlloc = new PeriodicRLESparseResourceAllocation(resCalc, periodicRle.getTimePeriod()); userPeriodicResourceAlloc.put(user, resAlloc); } } else { if (userResourceAlloc.containsKey(user)) { resAlloc = userResourceAlloc.get(user); } else { resAlloc = new RLESparseResourceAllocation(resCalc); userResourceAlloc.put(user, resAlloc); } } return resAlloc; }
InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, Map<ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Resource minAlloc) { this.contract = contract; this.startTime = startTime; this.endTime = endTime; this.reservationID = reservationID; this.user = user; this.allocationRequests = allocationRequests; this.planName = planName; resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc); for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests .entrySet()) { resourcesOverTime.addInterval(r.getKey(), r.getValue()); if (r.getValue().getConcurrency() > 1) { hasGang = true; } } }
InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, Map<ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Resource minAlloc) { this.contract = contract; this.startTime = startTime; this.endTime = endTime; this.reservationID = reservationID; this.user = user; this.allocationRequests = allocationRequests; this.planName = planName; resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc); for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests .entrySet()) { resourcesOverTime.addInterval(r.getKey(), r.getValue()); if (r.getValue().getConcurrency() > 1) { hasGang = true; } } }
private void incrementAllocation(ReservationAllocation reservation) { assert (readWriteLock.isWriteLockedByCurrentThread()); Map<ReservationInterval, ReservationRequest> allocationRequests = reservation.getAllocationRequests(); // check if we have encountered the user earlier and if not add an entry String user = reservation.getUser(); RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user); if (resAlloc == null) { resAlloc = new RLESparseResourceAllocation(resCalc, minAlloc); userResourceAlloc.put(user, resAlloc); } for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests .entrySet()) { resAlloc.addInterval(r.getKey(), r.getValue()); rleSparseVector.addInterval(r.getKey(), r.getValue()); } }
private void incrementAllocation(ReservationAllocation reservation) { assert (readWriteLock.isWriteLockedByCurrentThread()); Map<ReservationInterval, ReservationRequest> allocationRequests = reservation.getAllocationRequests(); // check if we have encountered the user earlier and if not add an entry String user = reservation.getUser(); RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user); if (resAlloc == null) { resAlloc = new RLESparseResourceAllocation(resCalc, minAlloc); userResourceAlloc.put(user, resAlloc); } for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests .entrySet()) { resAlloc.addInterval(r.getKey(), r.getValue()); rleSparseVector.addInterval(r.getKey(), r.getValue()); } }
@SuppressWarnings("checkstyle:parameternumber") public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy, ReservationAgent agent, Resource totalCapacity, long step, ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc, String queueName, Planner replanner, boolean getMoveOnExpiry, long maxPeriodicty, RMContext rmContext, Clock clock) { this.queueMetrics = queueMetrics; this.policy = policy; this.agent = agent; this.step = step; this.totalCapacity = totalCapacity; this.resCalc = resCalc; this.minAlloc = minAlloc; this.maxAlloc = maxAlloc; this.rleSparseVector = new RLESparseResourceAllocation(resCalc); this.maxPeriodicity = maxPeriodicty; this.periodicRle = new PeriodicRLESparseResourceAllocation(resCalc, this.maxPeriodicity); this.queueName = queueName; this.replanner = replanner; this.getMoveOnExpiry = getMoveOnExpiry; this.clock = clock; this.rmStateStore = rmContext.getStateStore(); }
public static RLESparseResourceAllocation generateRLESparseResourceAllocation( int[] alloc, long[] timeSteps) { TreeMap<Long, Resource> allocationsMap = new TreeMap<>(); for (int i = 0; i < alloc.length; i++) { allocationsMap.put(timeSteps[i], Resource.newInstance(alloc[i], alloc[i])); } RLESparseResourceAllocation rleVector = new RLESparseResourceAllocation( allocationsMap, new DefaultResourceCalculator()); return rleVector; }
/** * Merges the range start to end of two {@code RLESparseResourceAllocation} * using a given {@code RLEOperator}. * * @param resCalc the resource calculator * @param clusterResource the total cluster resources (for DRF) * @param a the left operand * @param b the right operand * @param operator the operator to be applied during merge * @param start the start-time of the range to be considered * @param end the end-time of the range to be considered * @return the a merged RLESparseResourceAllocation, produced by applying * "operator" to "a" and "b" * @throws PlanningException in case the operator is subtractTestPositive and * the result would contain a negative value */ public static RLESparseResourceAllocation merge(ResourceCalculator resCalc, Resource clusterResource, RLESparseResourceAllocation a, RLESparseResourceAllocation b, RLEOperator operator, long start, long end) throws PlanningException { NavigableMap<Long, Resource> cumA = a.getRangeOverlapping(start, end).getCumulative(); NavigableMap<Long, Resource> cumB = b.getRangeOverlapping(start, end).getCumulative(); NavigableMap<Long, Resource> out = merge(resCalc, clusterResource, cumA, cumB, start, end, operator); return new RLESparseResourceAllocation(out, resCalc); }
@Test public void testMergeMin() throws PlanningException { TreeMap<Long, Resource> a = new TreeMap<>(); TreeMap<Long, Resource> b = new TreeMap<>(); setupArrays(a, b); RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator()); RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator()); RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.min, 0, 60); System.out.println(out); long[] time = { 10, 22, 33, 40, 43, 50, 60 }; int[] alloc = { 5, 10, 15, 20, 10, 0 }; validate(out, time, alloc); }
@Test public void testMergeMax() throws PlanningException { TreeMap<Long, Resource> a = new TreeMap<>(); TreeMap<Long, Resource> b = new TreeMap<>(); setupArrays(a, b); RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator()); RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator()); RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.max, 0, 60); System.out.println(out); long[] time = { 10, 20, 30, 40, 50, 60 }; int[] alloc = { 5, 10, 15, 20, 10 }; validate(out, time, alloc); }
@Test public void testMergeSubtract() throws PlanningException { TreeMap<Long, Resource> a = new TreeMap<>(); TreeMap<Long, Resource> b = new TreeMap<>(); setupArrays(a, b); RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator()); RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator()); RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.subtract, 0, 60); System.out.println(out); long[] time = { 10, 11, 20, 22, 30, 33, 43, 50, 60 }; int[] alloc = { 5, 0, 5, 0, 5, 0, 10, -10 }; validate(out, time, alloc); }
@Test public void testMergeAdd() throws PlanningException { TreeMap<Long, Resource> a = new TreeMap<>(); TreeMap<Long, Resource> b = new TreeMap<>(); setupArrays(a, b); RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator()); RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator()); RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.add, 18, 45); System.out.println(out); long[] time = { 18, 20, 22, 30, 33, 40, 43, 45 }; int[] alloc = { 10, 15, 20, 25, 30, 40, 30 }; validate(out, time, alloc); }
@Test public void testZeroAllocation() { ResourceCalculator resCalc = new DefaultResourceCalculator(); RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc); rleSparseVector.addInterval(new ReservationInterval(0, Long.MAX_VALUE), Resource.newInstance(0, 0)); LOG.info(rleSparseVector.toString()); Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(new Random().nextLong())); Assert.assertTrue(rleSparseVector.isEmpty()); }
@Test public void testRangeOverlapping() { ResourceCalculator resCalc = new DefaultResourceCalculator(); RLESparseResourceAllocation r = new RLESparseResourceAllocation(resCalc); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; Set<Entry<ReservationInterval, Resource>> inputs = generateAllocation(start, alloc, false).entrySet(); for (Entry<ReservationInterval, Resource> ip : inputs) { r.addInterval(ip.getKey(), ip.getValue()); } long s = r.getEarliestStartTime(); long d = r.getLatestNonNullTime(); // tries to trigger "out-of-range" bug r = r.getRangeOverlapping(s, d); r = r.getRangeOverlapping(s - 1, d - 1); r = r.getRangeOverlapping(s + 1, d + 1); }
@Test public void testZeroAlloaction() { ResourceCalculator resCalc = new DefaultResourceCalculator(); Resource minAlloc = Resource.newInstance(1, 1); RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc); rleSparseVector.addInterval(new ReservationInterval(0, Long.MAX_VALUE), ReservationRequest.newInstance(Resource.newInstance(0, 0), (0))); LOG.info(rleSparseVector.toString()); Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(new Random().nextLong())); Assert.assertTrue(rleSparseVector.isEmpty()); }
protected void initialize(Plan plan, ReservationId reservationId, ReservationDefinition reservation) throws PlanningException { // Get plan step & capacity capacity = plan.getTotalCapacity(); step = plan.getStep(); // Get job parameters (type, arrival time & deadline) jobType = reservation.getReservationRequests().getInterpreter(); jobArrival = stepRoundUp(reservation.getArrival(), step); jobDeadline = stepRoundDown(reservation.getDeadline(), step); // Initialize the plan modifications planModifications = new RLESparseResourceAllocation(plan.getResourceCalculator()); // Dirty read of plan load // planLoads are not used by other StageAllocators... and don't deal // well with huge reservation ranges planLoads = plan.getCumulativeLoadOverTime(jobArrival, jobDeadline); ReservationAllocation oldRes = plan.getReservationById(reservationId); if (oldRes != null) { planLoads = RLESparseResourceAllocation.merge( plan.getResourceCalculator(), plan.getTotalCapacity(), planLoads, oldRes.getResourcesOverTime(jobArrival, jobDeadline), RLEOperator.subtract, jobArrival, jobDeadline); } }
private RLESparseResourceAllocation generateRLEAlloc(long period) { RLESparseResourceAllocation rle = new RLESparseResourceAllocation(new DefaultResourceCalculator()); Resource alloc = Resources.multiply(minAlloc, height * totCont); // loop in case the periodicity of the reservation is smaller than LCM long rStart = initTime % period; long rEnd = initTime % period + duration; // handle wrap-around if (period > 1 && rEnd > period) { long diff = rEnd - period; rEnd = period; // handle multiple wrap-arounds (e.g., 5h duration on a 2h periodicity) if(duration > period) { rle.addInterval(new ReservationInterval(0, period), Resources.multiply(alloc, duration / period - 1)); rle.addInterval(new ReservationInterval(0, diff % period), alloc); } else { rle.addInterval(new ReservationInterval(0, diff), alloc); } } if(rStart > rEnd){ rle.addInterval(new ReservationInterval(rStart, period), alloc); rle.addInterval(new ReservationInterval(0, rEnd), alloc); } else { rle.addInterval(new ReservationInterval(rStart, rEnd), alloc); } return rle; }