public DictColDeduper(int enableThresholdMB, int resetThresholdMB) { // only enable when there is sufficient memory this.enabled = MemoryBudgetController.getSystemAvailMB() >= enableThresholdMB; this.resetThresholdMB = resetThresholdMB; }
public void markHigh() { // get avail mem without gc int mb = MemoryBudgetController.getSystemAvailMB(); if (mb < lowAvail) { lowAvail = mb; logger.warn("Lower system avail " + lowAvail + " MB in markHigh()"); } }
public MemoryBudgetController(int totalBudgetMB) { Preconditions.checkArgument(totalBudgetMB >= 0); Preconditions.checkState(totalBudgetMB <= getSystemAvailMB()); this.totalBudgetMB = totalBudgetMB; this.totalReservedMB = 0; }
public void resetIfShortOfMem() { if (MemoryBudgetController.getSystemAvailMB() < resetThresholdMB) { for (Set<String> set : colValueSets.values()) set.clear(); } }
private int calculateReserveMB(Configuration configuration) { int sysAvailMB = MemoryBudgetController.getSystemAvailMB(); int mrReserve = configuration.getInt("mapreduce.task.io.sort.mb", 100); int sysReserve = Math.max(sysAvailMB / 10, 100); int reserveMB = mrReserve + sysReserve; logger.info("Reserve " + reserveMB + " MB = " + mrReserve + " (MR reserve) + " + sysReserve + " (SYS reserve)"); return reserveMB; }
private boolean shouldCutSplit(int nSplit, long splitRowCount) { int systemAvailMB = MemoryBudgetController.getSystemAvailMB(); logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB + " MB left, " + reserveMemoryMB + " MB threshold"); if (splitRowCount >= splitRowThreshold) { logger.info("Split cut due to hitting splitRowThreshold " + splitRowThreshold); return true; } if (systemAvailMB <= reserveMemoryMB) { logger.info("Split cut due to hitting memory threshold, system avail " + systemAvailMB + " MB <= reserve " + reserveMemoryMB + " MB"); return true; } return false; }
public static int gcAndGetSystemAvailMB() { final int tolerance = 1; try { int lastMB = -1; while (true) { Runtime.getRuntime().gc(); Thread.sleep(1000); int thisMB = getSystemAvailMB(); if (lastMB < 0) { lastMB = thisMB; continue; } if (lastMB - thisMB < tolerance) { return thisMB; } lastMB = thisMB; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); logger.error("", e); return getSystemAvailMB(); } }
@Override final protected void cleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { try { logger.info("Do cleanup, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); doCleanup(context); logger.info("Total rows: {}", mapCounter); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (InterruptedException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (RuntimeException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (Error ex) { // KYLIN-2170 logger.error("", ex); throw ex; } }
@Override final protected void setup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { try { logger.info("Do setup, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); doSetup(context); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (InterruptedException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (RuntimeException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (Error ex) { // KYLIN-2170 logger.error("", ex); throw ex; } }
@Override final protected void setup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { try { logger.info("Do setup, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); doSetup(context); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (InterruptedException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (RuntimeException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (Error ex) { // KYLIN-2170 logger.error("", ex); throw ex; } }
@Override final protected void cleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { try { logger.info("Do cleanup, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); doCleanup(context); logger.info("Total rows: " + reduceCounter); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (InterruptedException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (RuntimeException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (Error ex) { // KYLIN-2170 logger.error("", ex); throw ex; } }
@Override final public void map(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { try { if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Accepting Mapper Key with ordinal: " + mapCounter); logger.info("Do map, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); } doMap(key, value, context); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (InterruptedException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (RuntimeException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (Error ex) { // KYLIN-2170 logger.error("", ex); throw ex; } }
@Override final public void reduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { try { if (reduceCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Accepting Reducer Key with ordinal: " + reduceCounter); logger.info("Do reduce, available memory: {}m", MemoryBudgetController.getSystemAvailMB()); } doReduce(key, values, context); } catch (IOException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (InterruptedException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (RuntimeException ex) { // KYLIN-2170 logger.error("", ex); throw ex; } catch (Error ex) { // KYLIN-2170 logger.error("", ex); throw ex; } }
@Test public void test() { final int n = MemoryBudgetController.getSystemAvailMB() / 2; final MemoryBudgetController mbc = new MemoryBudgetController(n);
public DictColDeduper(int enableThresholdMB, int resetThresholdMB) { // only enable when there is sufficient memory this.enabled = MemoryBudgetController.getSystemAvailMB() >= enableThresholdMB; this.resetThresholdMB = resetThresholdMB; }
public void markHigh() { // get avail mem without gc int mb = MemoryBudgetController.getSystemAvailMB(); if (mb < lowAvail) { lowAvail = mb; logger.warn("Lower system avail " + lowAvail + " MB in markHigh()"); } }
public void resetIfShortOfMem() { if (MemoryBudgetController.getSystemAvailMB() < resetThresholdMB) { for (Set<String> set : colValueSets.values()) set.clear(); } }
public MemoryBudgetController(int totalBudgetMB) { Preconditions.checkArgument(totalBudgetMB >= 0); Preconditions.checkState(totalBudgetMB <= getSystemAvailMB()); this.totalBudgetMB = totalBudgetMB; this.totalReservedMB = 0; }
private boolean shouldCutSplit(int nSplit, long splitRowCount) { int systemAvailMB = MemoryBudgetController.getSystemAvailMB(); logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB + " MB left, " + reserveMemoryMB + " MB threshold"); if (splitRowCount >= splitRowThreshold) { logger.info("Split cut due to hitting splitRowThreshold " + splitRowThreshold); return true; } if (systemAvailMB <= reserveMemoryMB) { logger.info("Split cut due to hitting memory threshold, system avail " + systemAvailMB + " MB <= reserve " + reserveMemoryMB + " MB"); return true; } return false; }
private int calculateReserveMB(Configuration configuration) { int sysAvailMB = MemoryBudgetController.getSystemAvailMB(); int mrReserve = configuration.getInt("mapreduce.task.io.sort.mb", 100); int sysReserve = Math.max(sysAvailMB / 10, 100); int reserveMB = mrReserve + sysReserve; logger.info("Reserve " + reserveMB + " MB = " + mrReserve + " (MR reserve) + " + sysReserve + " (SYS reserve)"); return reserveMB; }