@Override public Page getOutput() { if (state != State.HAS_OUTPUT) { return null; } // project results into output blocks List<Type> types = aggregates.stream().map(Aggregator::getType).collect(toImmutableList()); // output page will only be constructed once, // so a new PageBuilder is constructed (instead of using PageBuilder.reset) PageBuilder pageBuilder = new PageBuilder(1, types); pageBuilder.declarePosition(); for (int i = 0; i < aggregates.size(); i++) { Aggregator aggregator = aggregates.get(i); BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i); aggregator.evaluate(blockBuilder); } state = State.FINISHED; return pageBuilder.build(); } }
public void flush(boolean force) { // add all full pages to output buffer for (int partition = 0; partition < pageBuilders.length; partition++) { PageBuilder partitionPageBuilder = pageBuilders[partition]; if (!partitionPageBuilder.isEmpty() && (force || partitionPageBuilder.isFull())) { Page pagePartition = partitionPageBuilder.build(); partitionPageBuilder.reset(); List<SerializedPage> serializedPages = splitPage(pagePartition, DEFAULT_MAX_PAGE_SIZE_IN_BYTES).stream() .map(serde::serialize) .collect(toImmutableList()); outputBuffer.enqueue(partition, serializedPages); pagesAdded.incrementAndGet(); rowsAdded.addAndGet(pagePartition.getPositionCount()); } } } }
public static void appendPositionTo(Page page, int position, PageBuilder pageBuilder) { pageBuilder.declarePosition(); for (int i = 0; i < page.getChannelCount(); i++) { Type type = pageBuilder.getType(i); Block block = page.getBlock(i); BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i); type.appendTo(block, position, blockBuilder); } }
public boolean isFull() { return estimatedProbeBlockBytes + buildPageBuilder.getSizeInBytes() >= DEFAULT_MAX_PAGE_SIZE_IN_BYTES || buildPageBuilder.isFull(); }
this.outputChannels = Ints.toArray(outputChannels); this.windowFunctions = windowFunctionDefinitions.stream() .map(functionDefinition -> new FramedWindowFunction(functionDefinition.createWindowFunction(), functionDefinition.getFrameInfo())) .collect(toImmutableList()); List<Type> types = Stream.concat( outputChannels.stream() .map(sourceTypes::get), windowFunctionDefinitions.stream() .map(WindowFunctionDefinition::getType)) .collect(toImmutableList()); List<Integer> unGroupedPartitionChannels = partitionChannels.stream() .filter(channel -> !preGroupedChannels.contains(channel)) .collect(toImmutableList()); this.unGroupedPartitionHashStrategy = pagesIndex.createPagesHashStrategy(unGroupedPartitionChannels, OptionalInt.empty()); List<Integer> preSortedChannels = sortChannels.stream() this.peerGroupHashStrategy = pagesIndex.createPagesHashStrategy(sortChannels, OptionalInt.empty()); this.pageBuilder = new PageBuilder(types);
public StreamingAggregationOperator(OperatorContext operatorContext, List<Type> sourceTypes, List<Type> groupByTypes, List<Integer> groupByChannels, Step step, List<AccumulatorFactory> accumulatorFactories, JoinCompiler joinCompiler) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(StreamingAggregationOperator.class.getSimpleName()); this.userMemoryContext = operatorContext.localUserMemoryContext(); this.groupByTypes = ImmutableList.copyOf(requireNonNull(groupByTypes, "groupByTypes is null")); this.groupByChannels = Ints.toArray(requireNonNull(groupByChannels, "groupByChannels is null")); this.accumulatorFactories = requireNonNull(accumulatorFactories, "accumulatorFactories is null"); this.step = requireNonNull(step, "step is null"); this.aggregates = setupAggregates(step, accumulatorFactories); this.pageBuilder = new PageBuilder(toTypes(groupByTypes, aggregates)); requireNonNull(joinCompiler, "joinCompiler is null"); requireNonNull(sourceTypes, "sourceTypes is null"); pagesHashStrategy = joinCompiler.compilePagesHashStrategyFactory(sourceTypes, groupByChannels, Optional.empty()) .createPagesHashStrategy( sourceTypes.stream() .map(type -> ImmutableList.<Block>of()) .collect(toImmutableList()), OptionalInt.empty()); }
public MergingPageIterator( Collection<Iterator<Page>> iterators, List<Type> types, List<Integer> sortFields, List<SortOrder> sortOrders) { requireNonNull(sortFields, "sortFields is null"); requireNonNull(sortOrders, "sortOrders is null"); checkArgument(sortFields.size() == sortOrders.size(), "sortFields and sortOrders size must match"); this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); this.sortFields = ImmutableList.copyOf(sortFields); this.sortOrders = ImmutableList.copyOf(sortOrders); this.pageBuilder = new PageBuilder(types); this.pagePositions = mergeSorted( iterators.stream() .map(pages -> concat(transform(pages, PagePositionIterator::new))) .collect(toList()), naturalOrder()); }
PageBuilder pageBuilder = new PageBuilder(columnTypes); int rows = 0; for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) { break; pageBuilder.declarePosition(); for (int i = 0; i < columns.size(); i++) { LineItemColumn column = columns.get(i); BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i); switch (column.getType().getBase()) { case IDENTIFIER: BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem)); break; case INTEGER: Page page = pageBuilder.build(); pageSink.appendPage(page); getFutureValue(pageSink.finish()); List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc"))); File outputFile = getOnlyElement(files); long length = outputFile.length(); Page nextPage = pageSource.getNextPage(); if (nextPage != null) { pages.add(nextPage.getLoadedPage());
@Setup public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder(); Block[] blocks = new Block[TYPES.size()]; Type returnType = new ArrayType(BOOLEAN); for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); Signature signature = new Signature("transform", FunctionKind.SCALAR, returnType.getTypeSignature(), arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)")); Signature greaterThan = new Signature("$operator$" + GREATER_THAN.name(), FunctionKind.SCALAR, BOOLEAN.getTypeSignature(), BIGINT.getTypeSignature(), BIGINT.getTypeSignature()); projectionsBuilder.add(new CallExpression(signature, returnType, ImmutableList.of( new InputReferenceExpression(0, arrayType), new LambdaDefinitionExpression( ImmutableList.of(BIGINT), ImmutableList.of("x"), new CallExpression(greaterThan, BOOLEAN, ImmutableList.of(new VariableReferenceExpression("x", BIGINT), new ConstantExpression(0L, BIGINT))))))); blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); } ImmutableList<RowExpression> projections = projectionsBuilder.build(); pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); pageBuilder = new PageBuilder(projections.stream().map(RowExpression::getType).collect(Collectors.toList())); page = new Page(blocks); }
private static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList()); List<Type> columnTypes = columns.stream().map(HiveFileFormatBenchmark::getColumnType) .map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()) .collect(toList()); PageBuilder pageBuilder = new PageBuilder(columnTypes); ImmutableList.Builder<Page> pages = ImmutableList.builder(); long dataSize = 0; for (E row : tpchTable.createGenerator(10, 1, 1)) { pageBuilder.declarePosition(); for (int i = 0; i < columns.size(); i++) { TpchColumn<E> column = columns.get(i); BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i); switch (column.getType().getBase()) { case IDENTIFIER: BIGINT.writeLong(blockBuilder, column.getIdentifier(row)); break; case INTEGER: if (pageBuilder.isFull()) { Page page = pageBuilder.build(); pages.add(page); pageBuilder.reset(); dataSize += page.getSizeInBytes(); return new TestData(columnNames, columnTypes, pages.build());
@Benchmark public List<Page> runPagesMergeSortBenchmark(MergeSortedBenchmarkData data) { WorkProcessor<Page> sortedPagesWork = mergeSortedPages( data.getSplitPages().stream() .map(WorkProcessor::fromIterable) .collect(toImmutableList()), ORDERING_COMPILER.compilePageWithPositionComparator(data.getSortTypes(), data.getSortChannels(), data.getSortOrders()), data.getOutputChannels(), data.getTypes(), (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, newSimpleAggregatedMemoryContext(), new DriverYieldSignal()); ImmutableList.Builder<Page> sortedPages = ImmutableList.builder(); while (true) { sortedPagesWork.process(); if (sortedPagesWork.isFinished()) { return sortedPages.build(); } sortedPages.add(sortedPagesWork.getResult()); } }
public SpatialJoinOperator( OperatorContext operatorContext, SpatialJoinNode.Type joinType, List<Type> probeTypes, List<Integer> probeOutputChannels, int probeGeometryChannel, Optional<Integer> partitionChannel, PagesSpatialIndexFactory pagesSpatialIndexFactory) { this.operatorContext = operatorContext; this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.joinType = joinType; this.probeTypes = ImmutableList.copyOf(probeTypes); this.probeOutputChannels = ImmutableList.copyOf(probeOutputChannels); this.probeGeometryChannel = probeGeometryChannel; this.partitionChannel = requireNonNull(partitionChannel, "partitionChannel is null"); this.pagesSpatialIndexFactory = pagesSpatialIndexFactory; this.pagesSpatialIndexFuture = pagesSpatialIndexFactory.createPagesSpatialIndex(); this.pageBuilder = new PageBuilder(ImmutableList.<Type>builder() .addAll(probeOutputChannels.stream() .map(probeTypes::get) .iterator()) .addAll(pagesSpatialIndexFactory.getOutputTypes()) .build()); }
public static WorkProcessor<Page> mergeSortedPages( List<WorkProcessor<Page>> pageProducers, PageWithPositionComparator comparator, List<Type> outputTypes, AggregatedMemoryContext aggregatedMemoryContext, DriverYieldSignal yieldSignal) { return mergeSortedPages( pageProducers, comparator, IntStream.range(0, outputTypes.size()).boxed().collect(toImmutableList()), outputTypes, (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, aggregatedMemoryContext, yieldSignal); }
private static List<Page> createBigintPages(int positionCount, int groupCount, int channelCount, boolean hashEnabled) { List<Type> types = Collections.nCopies(channelCount, BIGINT); ImmutableList.Builder<Page> pages = ImmutableList.builder(); if (hashEnabled) { types = ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BIGINT))); } PageBuilder pageBuilder = new PageBuilder(types); for (int position = 0; position < positionCount; position++) { int rand = ThreadLocalRandom.current().nextInt(groupCount); pageBuilder.declarePosition(); for (int numChannel = 0; numChannel < channelCount; numChannel++) { BIGINT.writeLong(pageBuilder.getBlockBuilder(numChannel), rand); } if (hashEnabled) { BIGINT.writeLong(pageBuilder.getBlockBuilder(channelCount), BigintOperators.hashCode(rand)); } if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pages.build(); }
@UsedByGeneratedCode public static Object createState(MapType mapType) { return new PageBuilder(ImmutableList.of(mapType)); }
hashChannelBuilder.add(TypeUtils.getHashBlock(joinTypes, varcharChannel.get(i), longChannel.get(i), doubleChannel.get(i), booleanChannel.get(i))); precomputedHash = hashChannelBuilder.build(); PageBuilder pageBuilder = new PageBuilder(outputTypes); leftBlocks[3] = booleanChannel.get(leftBlockIndex); int leftPositionCount = varcharChannel.get(leftBlockIndex).getPositionCount(); for (int leftBlockPosition = 0; leftBlockPosition < leftPositionCount; leftBlockPosition++) { for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) { assertEquals( hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), rightBlocks[3] = booleanChannel.get(rightBlockIndex); int rightPositionCount = varcharChannel.get(rightBlockIndex).getPositionCount(); for (int rightPosition = 0; rightPosition < rightPositionCount; rightPosition++) { boolean expected = expectedHashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)); assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected); assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlocks), rightPosition, new Page(rightBlocks)), expected); pageBuilder.declarePosition(); hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0); Page page = pageBuilder.build();
private Page createFragmentsPage() { Collection<Slice> fragments = getFutureValue(finishFuture); committed = true; updateWrittenBytes(); // output page will only be constructed once, // so a new PageBuilder is constructed (instead of using PageBuilder.reset) PageBuilder page = new PageBuilder(fragments.size() + 1, ImmutableList.of(types.get(ROW_COUNT_CHANNEL), types.get(FRAGMENT_CHANNEL))); BlockBuilder rowsBuilder = page.getBlockBuilder(0); BlockBuilder fragmentBuilder = page.getBlockBuilder(1); // write row count page.declarePosition(); BIGINT.writeLong(rowsBuilder, rowCount); fragmentBuilder.appendNull(); // write fragments for (Slice fragment : fragments) { page.declarePosition(); rowsBuilder.appendNull(); VARBINARY.writeSlice(fragmentBuilder, fragment); } return page.build(); }
public LookupOuterOperator( OperatorContext operatorContext, ListenableFuture<OuterPositionIterator> outerPositionsFuture, List<Type> probeOutputTypes, List<Type> buildOutputTypes, Runnable onClose) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.outerPositionsFuture = requireNonNull(outerPositionsFuture, "outerPositionsFuture is null"); List<Type> types = ImmutableList.<Type>builder() .addAll(requireNonNull(probeOutputTypes, "probeOutputTypes is null")) .addAll(requireNonNull(buildOutputTypes, "buildOutputTypes is null")) .build(); this.probeOutputTypes = ImmutableList.copyOf(probeOutputTypes); this.pageBuilder = new PageBuilder(types); this.onClose = requireNonNull(onClose, "onClose is null"); }