public DictColDeduper(int enableThresholdMB, int resetThresholdMB) { // only enable when there is sufficient memory this.enabled = MemoryBudgetController.getSystemAvailMB() >= enableThresholdMB; this.resetThresholdMB = resetThresholdMB; }
private CuboidResult buildCuboid(CuboidResult parent, long cuboidId) throws IOException { final String consumerName = "AggrCache@Cuboid " + cuboidId; MemoryBudgetController.MemoryConsumer consumer = new MemoryBudgetController.MemoryConsumer() { @Override public int freeUp(int mb) { return 0; // cannot free up on demand } @Override public String toString() { return consumerName; } }; // reserve memory for aggregation cache, can't be larger than the parent memBudget.reserveInsist(consumer, parent.aggrCacheMB); try { return aggregateCuboid(parent, cuboidId); } finally { memBudget.reserve(consumer, 0); } }
private void makeMemoryBudget() { int systemAvailMB = MemoryBudgetController.gcAndGetSystemAvailMB(); logger.info("System avail {} MB", systemAvailMB); int reserve = reserveMemoryMB; logger.info("Reserve {} MB for system basics", reserve); int budget = systemAvailMB - reserve; if (budget < baseResult.aggrCacheMB) { // make sure we have base aggr cache as minimal budget = baseResult.aggrCacheMB; logger.warn( "System avail memory ({} MB) is less than base aggr cache ({} MB) + minimal reservation ({} MB), consider increase JVM heap -Xmx", systemAvailMB, baseResult.aggrCacheMB, reserve); } logger.info("Memory Budget is {} MB", budget); memBudget = new MemoryBudgetController(budget); }
/** reserve without wait, fail with NotEnoughBudgetException immediately if no mem */ public void reserve(MemoryConsumer consumer, int requestMB) { if (totalBudgetMB == 0 && requestMB > 0) throw new NotEnoughBudgetException(); boolean ok = false; while (!ok) { int gap = calculateGap(consumer, requestMB); if (gap > 0) { // to void deadlock, don't hold lock when invoking consumer.freeUp() tryFreeUp(gap); } ok = updateBooking(consumer, requestMB); } }
@Test public void test() { final int n = MemoryBudgetController.getSystemAvailMB() / 2; final MemoryBudgetController mbc = new MemoryBudgetController(n); assertEquals(mbList.size(), mbc.getTotalReservedMB()); mbc.reserve(a, n); for (int i = 0; i < n; i++) { assertEquals(null, mbList.get(i).data); mbc.reserveInsist(b, n); assertTrue(System.currentTimeMillis() - bWaitStart > 1000); mbc.reserve(a, 1); fail(); } catch (NotEnoughBudgetException ex) {
public void clear() { chunkCount = 0; firstChunk = lastChunk = null; budgetCtrl.reserve(this, 0); }
public static int getSystemAvailMB() { return (int) (getSystemAvailBytes() / ONE_MB); }
public void markLow() { // get avail mem after gc int mb = MemoryBudgetController.gcAndGetSystemAvailMB(); if (mb > highAvail) { highAvail = mb; logger.warn("Higher system avail " + highAvail + " MB in markLow()"); } }
public void activateMemWrite() { if (budgetCtrl.getTotalBudgetMB() > 0) { writeActivated = true; if (debug) logger.debug(MemDiskStore.this + " mem write activated"); } }
Consumer(MemoryBudgetController mbc) { mbc.reserve(this, 1); data = new byte[MemoryBudgetController.ONE_MB - 24]; // 24 is object shell of this + object shell of data + reference of data }
/** reserve without wait, fail with NotEnoughBudgetException immediately if no mem */ public void reserve(MemoryConsumer consumer, int requestMB) { if (totalBudgetMB == 0 && requestMB > 0) throw new NotEnoughBudgetException(); boolean ok = false; while (!ok) { int gap = calculateGap(consumer, requestMB); if (gap > 0) { // to void deadlock, don't hold lock when invoking consumer.freeUp() tryFreeUp(gap); } ok = updateBooking(consumer, requestMB); } }
public void checkMemoryUsage() { if (firstKey == null) return; // about memory calculation, // http://seniorjava.wordpress.com/2013/09/01/java-objects-memory-size-reference/ if (rowMemBytes <= 0) { if (aggBufMap.size() > 0) { rowMemBytes = 0; MeasureAggregator[] measureAggregators = aggBufMap.get(firstKey); for (MeasureAggregator agg : measureAggregators) { rowMemBytes += agg.getMemBytesEstimate(); } } } int size = aggBufMap.size(); long memUsage = (40L + rowMemBytes) * size; if (memUsage > MEMORY_USAGE_CAP) { throw new RuntimeException("Kylin coprocessor memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abort coprocessor."); } //If less than 5% of max memory long avail = MemoryBudgetController.getSystemAvailBytes(); if (avail < (MEMOERY_MAX_BYTES / 20)) { throw new RuntimeException("Running Kylin coprocessor when too little memory is left. Abort coprocessor. Current available memory is " + avail + ". Max memory is " + MEMOERY_MAX_BYTES); } } }
public void markLow() { // get avail mem after gc int mb = MemoryBudgetController.gcAndGetSystemAvailMB(); if (mb > highAvail) { highAvail = mb; logger.warn("Higher system avail " + highAvail + " MB in markLow()"); } }
public void activateMemWrite() { if (budgetCtrl.getTotalBudgetMB() > 0) { writeActivated = true; if (debug) logger.debug(MemDiskStore.this + " mem write activated"); } }
public void markHigh() { // get avail mem without gc int mb = MemoryBudgetController.getSystemAvailMB(); if (mb < lowAvail) { lowAvail = mb; logger.warn("Lower system avail " + lowAvail + " MB in markHigh()"); } }
private CuboidResult buildCuboid(CuboidResult parent, long cuboidId) throws IOException { final String consumerName = "AggrCache@Cuboid " + cuboidId; MemoryBudgetController.MemoryConsumer consumer = new MemoryBudgetController.MemoryConsumer() { @Override public int freeUp(int mb) { return 0; // cannot free up on demand } @Override public String toString() { return consumerName; } }; // reserve memory for aggregation cache, can't be larger than the parent memBudget.reserveInsist(consumer, parent.aggrCacheMB); try { return aggregateCuboid(parent, cuboidId); } finally { memBudget.reserve(consumer, 0); } }
@Override public void run() { try { Thread.sleep(2000); } catch (InterruptedException e) { e.printStackTrace(); } mbc.reserve(a, 0); } }.start();
private void makeMemoryBudget() { baseResult.aggrCacheMB = Math.max(baseCuboidMemTracker.getEstimateMB(), 10); // 10 MB at minimal logger.debug("Base cuboid aggr cache is {} MB", baseResult.aggrCacheMB); int systemAvailMB = MemoryBudgetController.gcAndGetSystemAvailMB(); logger.debug("System avail {} MB", systemAvailMB); int reserve = reserveMemoryMB; logger.debug("Reserve {} MB for system basics", reserve); int budget = systemAvailMB - reserve; if (budget < baseResult.aggrCacheMB) { // make sure we have base aggr cache as minimal budget = baseResult.aggrCacheMB; logger.warn("System avail memory ({} MB) is less than base aggr cache ({} MB) + minimal reservation ({} MB), consider increase JVM heap -Xmx", systemAvailMB, baseResult.aggrCacheMB, reserve); } logger.debug("Memory Budget is {} MB", budget); memBudget = new MemoryBudgetController(budget); }
public static int getSystemAvailMB() { return (int) (getSystemAvailBytes() / ONE_MB); }
public MemoryBudgetController(int totalBudgetMB) { Preconditions.checkArgument(totalBudgetMB >= 0); Preconditions.checkState(totalBudgetMB <= getSystemAvailMB()); this.totalBudgetMB = totalBudgetMB; this.totalReservedMB = 0; }