public TableScanOperator( OperatorContext operatorContext, PlanNodeId planNodeId, PageSourceProvider pageSourceProvider, Iterable<ColumnHandle> columns) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(TableScanOperator.class.getSimpleName()); }
public FilterAndProjectOperator( OperatorContext operatorContext, PageProcessor processor, MergingPageOutput mergingOutput) { this.processor = requireNonNull(processor, "processor is null"); this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.pageProcessorMemoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(ScanFilterAndProjectOperator.class.getSimpleName()); this.outputMemoryContext = operatorContext.newLocalSystemMemoryContext(FilterAndProjectOperator.class.getSimpleName()); this.mergingOutput = requireNonNull(mergingOutput, "mergingOutput is null"); }
public AggregationOperator(OperatorContext operatorContext, Step step, List<AccumulatorFactory> accumulatorFactories, boolean useSystemMemory) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(AggregationOperator.class.getSimpleName()); this.userMemoryContext = operatorContext.localUserMemoryContext(); this.useSystemMemory = useSystemMemory; requireNonNull(step, "step is null"); // wrapper each function with an aggregator requireNonNull(accumulatorFactories, "accumulatorFactories is null"); ImmutableList.Builder<Aggregator> builder = ImmutableList.builder(); for (AccumulatorFactory accumulatorFactory : accumulatorFactories) { builder.add(new Aggregator(accumulatorFactory, step)); } aggregates = builder.build(); }
public TableWriterOperator( OperatorContext operatorContext, ConnectorPageSink pageSink, List<Integer> columnChannels, Operator statisticAggregationOperator, List<Type> types, boolean statisticsCpuTimerEnabled) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.pageSinkMemoryContext = operatorContext.newLocalSystemMemoryContext(TableWriterOperator.class.getSimpleName()); this.pageSink = requireNonNull(pageSink, "pageSink is null"); this.columnChannels = requireNonNull(columnChannels, "columnChannels is null"); this.operatorContext.setInfoSupplier(this::getInfo); this.statisticAggregationOperator = requireNonNull(statisticAggregationOperator, "statisticAggregationOperator is null"); this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); this.statisticsCpuTimerEnabled = statisticsCpuTimerEnabled; }
protected ScanFilterAndProjectOperator( OperatorContext operatorContext, PlanNodeId sourceId, PageSourceProvider pageSourceProvider, CursorProcessor cursorProcessor, PageProcessor pageProcessor, Iterable<ColumnHandle> columns, Iterable<Type> types, MergingPageOutput mergingOutput) { this.cursorProcessor = requireNonNull(cursorProcessor, "cursorProcessor is null"); this.pageProcessor = requireNonNull(pageProcessor, "pageProcessor is null"); this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.planNodeId = requireNonNull(sourceId, "sourceId is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.pageSourceMemoryContext = operatorContext.newLocalSystemMemoryContext(ScanFilterAndProjectOperator.class.getSimpleName()); this.pageProcessorMemoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(ScanFilterAndProjectOperator.class.getSimpleName()); this.outputMemoryContext = operatorContext.newLocalSystemMemoryContext(ScanFilterAndProjectOperator.class.getSimpleName()); this.mergingOutput = requireNonNull(mergingOutput, "mergingOutput is null"); this.pageBuilder = new PageBuilder(ImmutableList.copyOf(requireNonNull(types, "types is null"))); }
private WorkProcessor<Page> mergeSortedPages(WorkProcessor<Page> sortedPages, long memoryLimitForMerge) { merger = Optional.of(new MergingHashAggregationBuilder( accumulatorFactories, step, expectedGroups, groupByTypes, hashChannel, operatorContext, sortedPages, operatorContext.newLocalSystemMemoryContext(SpillableHashAggregationBuilder.class.getSimpleName()), memoryLimitForMerge, hashAggregationBuilder.getKeyChannels(), joinCompiler)); return merger.get().buildResult(); }
public StreamingAggregationOperator(OperatorContext operatorContext, List<Type> sourceTypes, List<Type> groupByTypes, List<Integer> groupByChannels, Step step, List<AccumulatorFactory> accumulatorFactories, JoinCompiler joinCompiler) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(StreamingAggregationOperator.class.getSimpleName()); this.userMemoryContext = operatorContext.localUserMemoryContext(); this.groupByTypes = ImmutableList.copyOf(requireNonNull(groupByTypes, "groupByTypes is null")); this.groupByChannels = Ints.toArray(requireNonNull(groupByChannels, "groupByChannels is null")); this.accumulatorFactories = requireNonNull(accumulatorFactories, "accumulatorFactories is null"); this.step = requireNonNull(step, "step is null"); this.aggregates = setupAggregates(step, accumulatorFactories); this.pageBuilder = new PageBuilder(toTypes(groupByTypes, aggregates)); requireNonNull(joinCompiler, "joinCompiler is null"); requireNonNull(sourceTypes, "sourceTypes is null"); pagesHashStrategy = joinCompiler.compilePagesHashStrategyFactory(sourceTypes, groupByChannels, Optional.empty()) .createPagesHashStrategy( sourceTypes.stream() .map(type -> ImmutableList.<Block>of()) .collect(toImmutableList()), OptionalInt.empty()); }
private ListenableFuture<?> spillIndex() { checkState(!spiller.isPresent(), "Spiller already created"); spiller = Optional.of(singleStreamSpillerFactory.create( index.getTypes(), operatorContext.getSpillContext().newLocalSpillContext(), operatorContext.newLocalSystemMemoryContext(HashBuilderOperator.class.getSimpleName()))); return getSpiller().spill(index.getPages()); }
public PartitionedOutputOperator( OperatorContext operatorContext, List<Type> sourceTypes, Function<Page, Page> pagePreprocessor, PartitionFunction partitionFunction, List<Integer> partitionChannels, List<Optional<NullableValue>> partitionConstants, boolean replicatesAnyRow, OptionalInt nullChannel, OutputBuffer outputBuffer, PagesSerdeFactory serdeFactory, DataSize maxMemory) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.pagePreprocessor = requireNonNull(pagePreprocessor, "pagePreprocessor is null"); this.partitionFunction = new PagePartitioner( partitionFunction, partitionChannels, partitionConstants, replicatesAnyRow, nullChannel, outputBuffer, serdeFactory, sourceTypes, maxMemory); operatorContext.setInfoSupplier(this::getInfo); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(PartitionedOutputOperator.class.getSimpleName()); this.partitionsInitialRetainedSize = this.partitionFunction.getRetainedSizeInBytes(); this.systemMemoryContext.setBytes(partitionsInitialRetainedSize); }
this.partial = step.isOutputPartial(); this.maxPartialMemory = maxPartialMemory.map(dataSize -> OptionalLong.of(dataSize.toBytes())).orElseGet(OptionalLong::empty); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(InMemoryHashAggregationBuilder.class.getSimpleName()); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.useSystemMemory = useSystemMemory;
@Test public void testLocalTotalMemoryLimitExceeded() { LocalMemoryContext systemMemoryContext = operatorContext.newLocalSystemMemoryContext("test"); systemMemoryContext.setBytes(100); assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 100, 0); systemMemoryContext.setBytes(queryMaxTotalMemory.toBytes()); assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, queryMaxTotalMemory.toBytes(), 0); try { systemMemoryContext.setBytes(queryMaxTotalMemory.toBytes() + 1); fail("allocation should hit the per-node total memory limit"); } catch (ExceededMemoryLimitException e) { assertEquals(e.getMessage(), format("Query exceeded per-node total memory limit of %1$s [Allocated: %1$s, Delta: 1B, Top Consumers: {test=%1$s}]", queryMaxTotalMemory)); } }