public MemoryManager(float ratio, long minAllocation) { checkRatio(ratio); memoryPoolRatio = ratio; minMemoryAllocation = minAllocation; totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax () * ratio); LOG.debug(String.format("Allocated total memory pool is: %,d", totalMemoryPool)); }
/** * {@inheritDoc} */ @Override public void close(TaskAttemptContext context) throws IOException, InterruptedException { internalWriter.close(); if (memoryManager != null) { memoryManager.removeWriter(internalWriter); } }
/** * Remove the given writer from the memory manager. * @param writer the writer that has been closed */ synchronized void removeWriter(InternalParquetRecordWriter writer) { if (writerList.containsKey(writer)) { writerList.remove(writer); } if (!writerList.isEmpty()) { updateAllocation(); } }
MemoryManager.DEFAULT_MIN_MEMORY_ALLOCATION); if (memoryManager == null) { memoryManager = new MemoryManager(maxLoad, minAllocation); } else if (memoryManager.getMemoryPoolRatio() != maxLoad) { LOG.warn("The configuration " + MEMORY_POOL_RATIO + " has been set. It should not " + "be reset by the new value: " + maxLoad);
validating, writerVersion); this.memoryManager = checkNotNull(memoryManager, "memoryManager"); memoryManager.addWriter(internalWriter, blockSize);
/** * Add a new writer and its memory allocation to the memory manager. * @param writer the new created writer * @param allocation the requested buffer size */ synchronized void addWriter(InternalParquetRecordWriter writer, Long allocation) { Long oldValue = writerList.get(writer); if (oldValue == null) { writerList.put(writer, allocation); } else { throw new IllegalArgumentException("[BUG] The Parquet Memory Manager should not add an " + "instance of InternalParquetRecordWriter more than once. The Manager already contains " + "the writer: " + writer); } updateAllocation(); }