public long getRetainedSizeInBytes() { return page.getRetainedSizeInBytes(); }
public FixedPageSource(Iterable<Page> pages) { this.pages = requireNonNull(pages, "pages is null").iterator(); long memoryUsageBytes = 0; for (Page page : pages) { memoryUsageBytes += page.getRetainedSizeInBytes(); } this.memoryUsageBytes = memoryUsageBytes; }
public long getEstimatedSizeInBytes() { return page.getRetainedSizeInBytes() + sizeOf(reference) + INSTANCE_SIZE; } }
public void addPage(Page page) { checkState(!finished, "NestedLoopJoinPagesBuilder is finished"); // ignore empty pages if (page.getPositionCount() == 0) { return; } pages.add(page); estimatedSize += page.getRetainedSizeInBytes(); }
@Override public long getSystemMemoryUsage() { long memoryUsage = 0; for (Page page : pages) { memoryUsage += page.getRetainedSizeInBytes(); } return memoryUsage; }
public long getRetainedSizeInBytes() { long retainedSizeInBytes = INSTANCE_SIZE; retainedSizeInBytes += pageBuilder.getRetainedSizeInBytes(); for (Page page : outputQueue) { retainedSizeInBytes += page.getRetainedSizeInBytes(); } return retainedSizeInBytes; } }
@Override public void accept(Page page) { memoryManager.updateMemoryUsage(page.getRetainedSizeInBytes()); PageReference pageReference = new PageReference(page, buffers.size(), () -> memoryManager.updateMemoryUsage(-page.getRetainedSizeInBytes())); for (Consumer<PageReference> buffer : buffers) { buffer.accept(pageReference); } }
@Override public void accept(Page page) { memoryManager.updateMemoryUsage(page.getRetainedSizeInBytes()); PageReference pageReference = new PageReference(page, 1, () -> memoryManager.updateMemoryUsage(-page.getRetainedSizeInBytes())); int randomIndex = ThreadLocalRandom.current().nextInt(buffers.size()); buffers.get(randomIndex).accept(pageReference); }
@Override public Work<?> addPage(Page page) { currentPageSizeInBytes = page.getRetainedSizeInBytes(); return new AddPageWork(page.getBlock(hashChannel)); }
public void add(Page page) { checkState(canAdd(page), "page buffer is full"); pages.add(page); usedMemoryBytes += page.getRetainedSizeInBytes(); rowCount = addExact(rowCount, page.getPositionCount()); }
@Override public Work<GroupByIdBlock> getGroupIds(Page page) { currentPageSizeInBytes = page.getRetainedSizeInBytes(); return new GetGroupIdsWork(page.getBlock(hashChannel)); }
public Page removePage() { checkNotHoldsLock(); // NOTE: there is no need to acquire a lock here. The buffer is concurrent // and buffered bytes is not expected to be consistent with the buffer (only // best effort). PageReference pageReference = buffer.poll(); if (pageReference == null) { return null; } // dereference the page outside of lock, since may trigger a callback Page page = pageReference.removePage(); bufferedBytes.addAndGet(-page.getRetainedSizeInBytes()); checkFinished(); return page; }
private void updateMemoryUsage() { long memorySize = pageBuilder.getRetainedSizeInBytes(); for (Page output : outputPages) { memorySize += output.getRetainedSizeInBytes(); } for (Aggregator aggregator : aggregates) { memorySize += aggregator.getEstimatedSize(); } if (currentGroup != null) { memorySize += currentGroup.getRetainedSizeInBytes(); } if (step.isOutputPartial()) { systemMemoryContext.setBytes(memorySize); } else { userMemoryContext.setBytes(memorySize); } }
public synchronized void add(Long tableId, Page page) { if (!contains(tableId)) { throw new PrestoException(MISSING_DATA, "Failed to find table on a worker."); } page.compact(); long newSize = currentBytes + page.getRetainedSizeInBytes(); if (maxBytes < newSize) { throw new PrestoException(MEMORY_LIMIT_EXCEEDED, format("Memory limit [%d] for memory connector exceeded", maxBytes)); } currentBytes = newSize; TableData tableData = tables.get(tableId); tableData.add(page); }
public void append(Page dataPage) { // getRegionSizeInBytes for each row can be expensive; use getRetainedSizeInBytes for estimation hiveWriterStats.addInputPageSizesInBytes(dataPage.getRetainedSizeInBytes()); fileWriter.appendRows(dataPage); rowCount += dataPage.getPositionCount(); inputSizeInBytes += dataPage.getSizeInBytes(); }
private static WorkProcessor<PageWithPosition> pageWithPositions(WorkProcessor<Page> pages, AggregatedMemoryContext aggregatedMemoryContext) { return pages.flatMap(page -> { LocalMemoryContext memoryContext = aggregatedMemoryContext.newLocalMemoryContext(MergeSortedPages.class.getSimpleName()); memoryContext.setBytes(page.getRetainedSizeInBytes()); return WorkProcessor.create(new WorkProcessor.Process<PageWithPosition>() { int position; @Override public ProcessState<PageWithPosition> process() { if (position >= page.getPositionCount()) { memoryContext.close(); return ProcessState.finished(); } return ProcessState.ofResult(new PageWithPosition(page, position++)); } }); }); }
public void compact() { if (getRetainedSizeInBytes() <= getSizeInBytes()) { return; } for (int i = 0; i < blocks.length; i++) { Block block = blocks[i]; if (block instanceof DictionaryBlock) { continue; } // Compact the block blocks[i] = block.copyRegion(0, block.getPositionCount()); } Map<DictionaryId, DictionaryBlockIndexes> dictionaryBlocks = getRelatedDictionaryBlocks(); for (DictionaryBlockIndexes blockIndexes : dictionaryBlocks.values()) { List<DictionaryBlock> compactBlocks = compactRelatedBlocks(blockIndexes.getBlocks()); List<Integer> indexes = blockIndexes.getIndexes(); for (int i = 0; i < compactBlocks.size(); i++) { blocks[indexes.get(i)] = compactBlocks.get(i); } } updateRetainedSize(); }
@Override public Work<GroupByIdBlock> getGroupIds(Page page) { currentPageSizeInBytes = page.getRetainedSizeInBytes(); if (isRunLengthEncoded(page)) { return new GetRunLengthEncodedGroupIdsWork(page); } if (canProcessDictionary(page)) { return new GetDictionaryGroupIdsWork(page); } return new GetNonDictionaryGroupIdsWork(page); }
@Override public Work<?> addPage(Page page) { currentPageSizeInBytes = page.getRetainedSizeInBytes(); if (isRunLengthEncoded(page)) { return new AddRunLengthEncodedPageWork(page); } if (canProcessDictionary(page)) { return new AddDictionaryPageWork(page); } return new AddNonDictionaryPageWork(page); }
@Override public void accept(Page page) { long retainedSizeInBytes = page.getRetainedSizeInBytes(); bufferMemoryManager.updateMemoryUsage(retainedSizeInBytes); memoryTracker.accept(retainedSizeInBytes); PageReference pageReference = new PageReference(page, 1, () -> { bufferMemoryManager.updateMemoryUsage(-retainedSizeInBytes); memoryTracker.accept(-retainedSizeInBytes); }); localExchangeSource.addPage(pageReference); }