@Override public Page getNextPage() { if (isFinished()) { return null; } Page page = pages.next(); completedBytes += page.getSizeInBytes(); return page; }
BlackHolePageSource(Page page, int count, ListeningScheduledExecutorService executorService, Duration pageProcessingDelay) { this.page = requireNonNull(page, "page is null"); checkArgument(count >= 0, "count is negative"); this.pagesLeft = count; this.executorService = requireNonNull(executorService, "executorService is null"); this.pageProcessingDelayInMillis = requireNonNull(pageProcessingDelay, "pageProcessingDelay is null").toMillis(); this.memoryUsageBytes = page.getSizeInBytes(); }
public boolean tryAddPage(Page page) { memoryInBytes += page.getSizeInBytes(); if (isMemoryExceeded()) { return false; } pages.add(page); return true; }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); checkState(!finished, "operator finished"); page = pagePreprocessor.apply(page); pageConsumer.accept(page); operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); }
public static long writePages(PagesSerde serde, SliceOutput sliceOutput, Iterator<Page> pages) { long size = 0; while (pages.hasNext()) { Page page = pages.next(); writeSerializedPage(sliceOutput, serde.serialize(page)); size += page.getSizeInBytes(); } return size; }
@Override public void addInput(Page page) { operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); page = pagePreprocessor.apply(page); sink.addPage(page); operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); }
private void process(Page page) { requireNonNull(page, "page is null"); // avoid memory copying for pages that are big enough if (page.getSizeInBytes() >= minPageSizeInBytes || page.getPositionCount() >= minRowCount) { flush(); outputQueue.add(page); return; } buffer(page); }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); checkState(blocked == NOT_BLOCKED, "output is already blocked"); ListenableFuture<?> future = pageBuffer.add(page); if (!future.isDone()) { this.blocked = future; } operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); checkState(!isFinished(), "Operator is already finished"); if (!indexSnapshotBuilder.tryAddPage(page)) { finish(); return; } operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); }
private static void assertPageSize(List<Page> pages, long maxPageSizeInBytes) { for (Page page : pages) { assertLessThanOrEqual(page.getSizeInBytes(), maxPageSizeInBytes); } }
@Override public Page getOutput() { Page page = source.removePage(); if (page != null) { operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); } return page; }
void recordAddInput(OperationTimer operationTimer, Page page) { operationTimer.recordOperationComplete(addInputTiming); if (page != null) { inputDataSize.update(page.getSizeInBytes()); inputPositions.update(page.getPositionCount()); } }
public void append(Page dataPage) { // getRegionSizeInBytes for each row can be expensive; use getRetainedSizeInBytes for estimation hiveWriterStats.addInputPageSizesInBytes(dataPage.getRetainedSizeInBytes()); fileWriter.appendRows(dataPage); rowCount += dataPage.getPositionCount(); inputSizeInBytes += dataPage.getSizeInBytes(); }
void recordGetOutput(OperationTimer operationTimer, Page page) { operationTimer.recordOperationComplete(getOutputTiming); if (page != null) { outputDataSize.update(page.getSizeInBytes()); outputPositions.update(page.getPositionCount()); } }
@Override public Page getOutput() { if (!mergedPages.process() || mergedPages.isFinished()) { return null; } Page page = mergedPages.getResult(); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); return page; }
@Override public Page getOutput() { if (closed || mergedPages == null || !mergedPages.process() || mergedPages.isFinished()) { return null; } Page page = mergedPages.getResult(); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); return page; }
@Override public Page getOutput() { SerializedPage page = exchangeClient.pollPage(); if (page == null) { return null; } operatorContext.recordRawInput(page.getSizeInBytes()); Page deserializedPage = serde.deserialize(page); operatorContext.recordProcessedInput(deserializedPage.getSizeInBytes(), page.getPositionCount()); return deserializedPage; }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); checkState(!isFinished(), "Operator is already finished"); index.addPage(page); if (!localUserMemoryContext.trySetBytes((index.getEstimatedSize().toBytes()))) { index.compact(); localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); } operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); }
@Test public void testMinPageSizeThreshold() { Page page = createSequencePage(TYPES, 10); MergingPageOutput output = new MergingPageOutput(TYPES, page.getSizeInBytes(), Integer.MAX_VALUE, Integer.MAX_VALUE); assertTrue(output.needsInput()); assertNull(output.getOutput()); output.addInput(createPagesIterator(page)); assertFalse(output.needsInput()); assertSame(output.getOutput(), page); }