public NestedLoopBuildOperator(OperatorContext operatorContext, NestedLoopJoinBridge nestedLoopJoinBridge) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.nestedLoopJoinBridge = requireNonNull(nestedLoopJoinBridge, "nestedLoopJoinBridge is null"); this.nestedLoopJoinPagesBuilder = new NestedLoopJoinPagesBuilder(operatorContext); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); }
public MarkDistinctOperator(OperatorContext operatorContext, List<Type> types, List<Integer> markDistinctChannels, Optional<Integer> hashChannel, JoinCompiler joinCompiler) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); requireNonNull(hashChannel, "hashChannel is null"); requireNonNull(markDistinctChannels, "markDistinctChannels is null"); ImmutableList.Builder<Type> distinctTypes = ImmutableList.builder(); for (int channel : markDistinctChannels) { distinctTypes.add(types.get(channel)); } this.markDistinctHash = new MarkDistinctHash(operatorContext.getSession(), distinctTypes.build(), Ints.toArray(markDistinctChannels), hashChannel, joinCompiler, this::updateMemoryReservation); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); }
public AggregationOperator(OperatorContext operatorContext, Step step, List<AccumulatorFactory> accumulatorFactories, boolean useSystemMemory) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(AggregationOperator.class.getSimpleName()); this.userMemoryContext = operatorContext.localUserMemoryContext(); this.useSystemMemory = useSystemMemory; requireNonNull(step, "step is null"); // wrapper each function with an aggregator requireNonNull(accumulatorFactories, "accumulatorFactories is null"); ImmutableList.Builder<Aggregator> builder = ImmutableList.builder(); for (AccumulatorFactory accumulatorFactory : accumulatorFactories) { builder.add(new Aggregator(accumulatorFactory, step)); } aggregates = builder.build(); }
public SpatialJoinOperator( OperatorContext operatorContext, SpatialJoinNode.Type joinType, List<Type> probeTypes, List<Integer> probeOutputChannels, int probeGeometryChannel, Optional<Integer> partitionChannel, PagesSpatialIndexFactory pagesSpatialIndexFactory) { this.operatorContext = operatorContext; this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.joinType = joinType; this.probeTypes = ImmutableList.copyOf(probeTypes); this.probeOutputChannels = ImmutableList.copyOf(probeOutputChannels); this.probeGeometryChannel = probeGeometryChannel; this.partitionChannel = requireNonNull(partitionChannel, "partitionChannel is null"); this.pagesSpatialIndexFactory = pagesSpatialIndexFactory; this.pagesSpatialIndexFuture = pagesSpatialIndexFactory.createPagesSpatialIndex(); this.pageBuilder = new PageBuilder(ImmutableList.<Type>builder() .addAll(probeOutputChannels.stream() .map(probeTypes::get) .iterator()) .addAll(pagesSpatialIndexFactory.getOutputTypes()) .build()); }
public OrderByOperator( OperatorContext operatorContext, List<Type> sourceTypes, List<Integer> outputChannels, int expectedPositions, List<Integer> sortChannels, List<SortOrder> sortOrder, PagesIndex.Factory pagesIndexFactory) { requireNonNull(pagesIndexFactory, "pagesIndexFactory is null"); this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.outputChannels = Ints.toArray(requireNonNull(outputChannels, "outputChannels is null")); this.sortChannels = ImmutableList.copyOf(requireNonNull(sortChannels, "sortChannels is null")); this.sortOrder = ImmutableList.copyOf(requireNonNull(sortOrder, "sortOrder is null")); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.pageIndex = pagesIndexFactory.newPagesIndex(sourceTypes, expectedPositions); this.pageBuilder = new PageBuilder(toTypes(sourceTypes, outputChannels)); }
public StreamingAggregationOperator(OperatorContext operatorContext, List<Type> sourceTypes, List<Type> groupByTypes, List<Integer> groupByChannels, Step step, List<AccumulatorFactory> accumulatorFactories, JoinCompiler joinCompiler) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(StreamingAggregationOperator.class.getSimpleName()); this.userMemoryContext = operatorContext.localUserMemoryContext(); this.groupByTypes = ImmutableList.copyOf(requireNonNull(groupByTypes, "groupByTypes is null")); this.groupByChannels = Ints.toArray(requireNonNull(groupByChannels, "groupByChannels is null")); this.accumulatorFactories = requireNonNull(accumulatorFactories, "accumulatorFactories is null"); this.step = requireNonNull(step, "step is null"); this.aggregates = setupAggregates(step, accumulatorFactories); this.pageBuilder = new PageBuilder(toTypes(groupByTypes, aggregates)); requireNonNull(joinCompiler, "joinCompiler is null"); requireNonNull(sourceTypes, "sourceTypes is null"); pagesHashStrategy = joinCompiler.compilePagesHashStrategyFactory(sourceTypes, groupByChannels, Optional.empty()) .createPagesHashStrategy( sourceTypes.stream() .map(type -> ImmutableList.<Block>of()) .collect(toImmutableList()), OptionalInt.empty()); }
public static final class SpatialIndexBuilderOperatorFactory implements OperatorFactory { private final int operatorId; private final PlanNodeId planNodeId; private final PagesSpatialIndexFactory pagesSpatialIndexFactory; private final List<Integer> outputChannels; private final int indexChannel; private final Optional<Integer> radiusChannel; private final Optional<Integer> partitionChannel; private final SpatialPredicate spatialRelationshipTest; private final Optional<JoinFilterFunctionFactory> filterFunctionFactory; private final PagesIndex.Factory pagesIndexFactory; private final Map<Integer, Rectangle> spatialPartitions = new HashMap<>(); private final int expectedPositions; private boolean closed; public SpatialIndexBuilderOperatorFactory( int operatorId, PlanNodeId planNodeId, List<Type> types, List<Integer> outputChannels, int indexChannel, Optional<Integer> radiusChannel, Optional<Integer> partitionChannel, SpatialPredicate spatialRelationshipTest, Optional<String> kdbTreeJson, Optional<JoinFilterFunctionFactory> filterFunctionFactory,
public TopNOperator( OperatorContext operatorContext, List<Type> types, int n, List<Integer> sortChannels, List<SortOrder> sortOrders) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); checkArgument(n >= 0, "n must be positive"); if (n == 0) { finishing = true; outputIterator = emptyIterator(); } else { topNBuilder = new GroupedTopNBuilder( types, new SimplePageWithPositionComparator(types, sortChannels, sortOrders), n, false, new NoChannelGroupByHash()); } }
this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.outputChannels = Ints.toArray(outputChannels); this.windowFunctions = windowFunctionDefinitions.stream()
this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.outputChannels = Ints.toArray(outputChannels); this.types = toTypes(sourceTypes, outputChannels);
this.localUserMemoryContext = operatorContext.localUserMemoryContext();
public ChannelSetBuilder(Type type, Optional<Integer> hashChannel, int expectedPositions, OperatorContext operatorContext, JoinCompiler joinCompiler) { List<Type> types = ImmutableList.of(type); this.hash = createGroupByHash( types, HASH_CHANNELS, hashChannel, expectedPositions, isDictionaryAggregationEnabled(operatorContext.getSession()), joinCompiler, this::updateMemoryReservation); this.nullBlockPage = new Page(type.createBlockBuilder(null, 1, UNKNOWN.getFixedSize()).appendNull().build()); this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.localMemoryContext = operatorContext.localUserMemoryContext(); }
private void closeAggregationBuilder() { outputPages = null; if (aggregationBuilder != null) { aggregationBuilder.recordHashCollisions(hashCollisionsCounter); aggregationBuilder.close(); // aggregationBuilder.close() will release all memory reserved in memory accounting. // The reference must be set to null afterwards to avoid unaccounted memory. aggregationBuilder = null; } operatorContext.localUserMemoryContext().setBytes(0); operatorContext.localRevocableMemoryContext().setBytes(0); }
this.sortChannel = sortChannel; this.searchFunctionFactories = searchFunctionFactories; this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.localRevocableMemoryContext = operatorContext.localRevocableMemoryContext();
public SpillableHashAggregationBuilder( List<AccumulatorFactory> accumulatorFactories, AggregationNode.Step step, int expectedGroups, List<Type> groupByTypes, List<Integer> groupByChannels, Optional<Integer> hashChannel, OperatorContext operatorContext, DataSize memoryLimitForMerge, DataSize memoryLimitForMergeWithMemory, SpillerFactory spillerFactory, JoinCompiler joinCompiler) { this.accumulatorFactories = accumulatorFactories; this.step = step; this.expectedGroups = expectedGroups; this.groupByTypes = groupByTypes; this.groupByChannels = groupByChannels; this.hashChannel = hashChannel; this.operatorContext = operatorContext; this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.localRevocableMemoryContext = operatorContext.localRevocableMemoryContext(); this.memoryLimitForMerge = memoryLimitForMerge.toBytes(); this.memoryLimitForMergeWithMemory = memoryLimitForMergeWithMemory.toBytes(); this.spillerFactory = spillerFactory; this.joinCompiler = joinCompiler; rebuildHashAggregationBuilder(); }
this.maxPartialMemory = maxPartialMemory.map(dataSize -> OptionalLong.of(dataSize.toBytes())).orElseGet(OptionalLong::empty); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(InMemoryHashAggregationBuilder.class.getSimpleName()); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.useSystemMemory = useSystemMemory;
@Test public void testTrySetZeroBytesFullPool() { LocalMemoryContext localMemoryContext = operatorContext.localUserMemoryContext(); // fill up the pool memoryPool.reserve(new QueryId("test_query"), "test", memoryPool.getFreeBytes()); // try to reserve 0 bytes in the full pool assertTrue(localMemoryContext.trySetBytes(localMemoryContext.getBytes())); }