@Override public boolean needsInput() { return !finishing && !pageBuilder.isFull() && currentPage == null; }
@Override public boolean needsInput() { return !pageBuilder.isFull(); }
@Override public void addInput(Page page) { checkState(!finishing, "Operator is already finishing"); requireNonNull(page, "page is null"); checkState(currentPage == null, "currentPage is not null"); checkState(!pageBuilder.isFull(), "Page buffer is full"); currentPage = page; currentPosition = 0; fillUnnesters(); }
public boolean isFull() { return estimatedProbeBlockBytes + buildPageBuilder.getSizeInBytes() >= DEFAULT_MAX_PAGE_SIZE_IN_BYTES || buildPageBuilder.isFull(); }
@Override public boolean needsInput() { return !finished && pagesSpatialIndexFuture.isDone() && !pageBuilder.isFull() && probe == null; }
public static WorkProcessor<Page> mergeSortedPages( List<WorkProcessor<Page>> pageProducers, PageWithPositionComparator comparator, List<Type> outputTypes, AggregatedMemoryContext aggregatedMemoryContext, DriverYieldSignal yieldSignal) { return mergeSortedPages( pageProducers, comparator, IntStream.range(0, outputTypes.size()).boxed().collect(toImmutableList()), outputTypes, (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, aggregatedMemoryContext, yieldSignal); }
public Builder add(Object... values) { pageBuilder.declarePosition(); for (int i = 0; i < types.size(); i++) { writeNativeValue(types.get(i), pageBuilder.getBlockBuilder(i), values[i]); } if (pageBuilder.isFull()) { flushPage(); } return this; }
@Override public Page getOutput() { // only return a page if the page buffer isFull or we are finishing and the page buffer has data if (pageBuilder.isFull() || (finishing && !pageBuilder.isEmpty())) { Page page = pageBuilder.build(); pageBuilder.reset(); return page; } return null; }
@Override protected Page computeNext() { while (!pageBuilder.isFull() && pagePositions.hasNext()) { pagePositions.next().appendTo(pageBuilder); } if (pageBuilder.isEmpty()) { return endOfData(); } Page page = pageBuilder.build(); pageBuilder.reset(); return page; }
public void flush(boolean force) { // add all full pages to output buffer for (int partition = 0; partition < pageBuilders.length; partition++) { PageBuilder partitionPageBuilder = pageBuilders[partition]; if (!partitionPageBuilder.isEmpty() && (force || partitionPageBuilder.isFull())) { Page pagePartition = partitionPageBuilder.build(); partitionPageBuilder.reset(); List<SerializedPage> serializedPages = splitPage(pagePartition, DEFAULT_MAX_PAGE_SIZE_IN_BYTES).stream() .map(serde::serialize) .collect(toImmutableList()); outputBuffer.enqueue(partition, serializedPages); pagesAdded.incrementAndGet(); rowsAdded.addAndGet(pagePartition.getPositionCount()); } } } }
@Override public void noMoreSplits() { mergedPages = mergeSortedPages( pageProducers, comparator, outputChannels, outputTypes, (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, operatorContext.aggregateUserMemoryContext(), operatorContext.getDriverContext().getYieldSignal()); blockedOnSplits.set(null); }
private void evaluateAndFlushGroup(Page page, int position) { pageBuilder.declarePosition(); for (int i = 0; i < groupByTypes.size(); i++) { Block block = page.getBlock(groupByChannels[i]); Type type = groupByTypes.get(i); type.appendTo(block, position, pageBuilder.getBlockBuilder(i)); } int offset = groupByTypes.size(); for (int i = 0; i < aggregates.size(); i++) { aggregates.get(i).evaluate(pageBuilder.getBlockBuilder(offset + i)); } if (pageBuilder.isFull()) { outputPages.add(pageBuilder.build()); pageBuilder.reset(); } aggregates = setupAggregates(step, accumulatorFactories); }
private Block computeResultBlock(Type type, Block block, int arrayLength) { if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); for (int i = 0; i < arrayLength; ++i) { type.appendTo(block, positions.get(i), blockBuilder); } pageBuilder.declarePositions(arrayLength); return blockBuilder.getRegion(blockBuilder.getPositionCount() - arrayLength, arrayLength); }
private void buffer(Page page) { pageBuilder.declarePositions(page.getPositionCount()); for (int channel = 0; channel < types.size(); channel++) { Type type = types.get(channel); for (int position = 0; position < page.getPositionCount(); position++) { type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel)); } } if (pageBuilder.isFull()) { flush(); } }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); checkState(!pageBuilder.isFull(), "Output buffer is full"); checkState(!finishing, "Operator is finished"); filterAndProjectRowOriented(pageBuilder, page.getBlock(0), page.getBlock(1), page.getBlock(2), page.getBlock(3), page.getBlock(4), page.getBlock(5), page.getBlock(6)); }
public int buildPage(int position, int[] outputChannels, PageBuilder pageBuilder) { while (!pageBuilder.isFull() && position < positionCount) { long pageAddress = valueAddresses.getLong(position); int blockIndex = decodeSliceIndex(pageAddress); int blockPosition = decodePosition(pageAddress); // append the row pageBuilder.declarePosition(); for (int i = 0; i < outputChannels.length; i++) { int outputChannel = outputChannels[i]; Type type = types.get(outputChannel); Block block = this.channels[outputChannel].get(blockIndex); type.appendTo(block, blockPosition, pageBuilder.getBlockBuilder(i)); } position++; } return position; }
private static BiPredicate<PageBuilder, PageWithPosition> keepSameHashValuesWithinSinglePage(InterpretedHashGenerator hashGenerator) { return (pageBuilder, pageWithPosition) -> { long hash = hashGenerator.hashPosition(pageWithPosition.getPosition(), pageWithPosition.getPage()); return !pageBuilder.isEmpty() && hashGenerator.hashPosition(pageBuilder.getPositionCount() - 1, pageBuilder::getBlockBuilder) != hash && pageBuilder.isFull(); }; }
private static Block featuresHelper(PageBuilder pageBuilder, double... features) { if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder mapBlockBuilder = pageBuilder.getBlockBuilder(0); BlockBuilder blockBuilder = mapBlockBuilder.beginBlockEntry(); for (int i = 0; i < features.length; i++) { BigintType.BIGINT.writeLong(blockBuilder, i); DoubleType.DOUBLE.writeDouble(blockBuilder, features[i]); } mapBlockBuilder.closeEntry(); pageBuilder.declarePosition(); return mapBlockBuilder.getObject(mapBlockBuilder.getPositionCount() - 1, Block.class); } }
@SqlType("row(x integer,y integer)") public Block bingTileCoordinates(@SqlType(BingTileType.NAME) long input) { if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); BingTile tile = BingTile.decode(input); BlockBuilder tileBlockBuilder = blockBuilder.beginBlockEntry(); INTEGER.writeLong(tileBlockBuilder, tile.getX()); INTEGER.writeLong(tileBlockBuilder, tile.getY()); blockBuilder.closeEntry(); pageBuilder.declarePosition(); return BING_TILE_COORDINATES_ROW_TYPE.getObject(blockBuilder, blockBuilder.getPositionCount() - 1); } }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object groupByHashPreCompute(BenchmarkData data) { GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), data.getHashChannel(), EXPECTED_SIZE, false, getJoinCompiler(data.isGroupByUsesEqual()), NOOP); data.getPages().forEach(p -> groupByHash.getGroupIds(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }