@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getGroupingKeys()); if (node.getHashSymbol().isPresent()) { expectedInputs.add(node.getHashSymbol().get()); } ImmutableMap.Builder<Symbol, Aggregation> aggregations = ImmutableMap.builder(); for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { Symbol symbol = entry.getKey(); if (context.get().contains(symbol)) { Aggregation aggregation = entry.getValue(); expectedInputs.addAll(SymbolsExtractor.extractUnique(aggregation.getCall())); aggregation.getMask().ifPresent(expectedInputs::add); aggregations.put(symbol, aggregation); } } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new AggregationNode(node.getId(), source, aggregations.build(), node.getGroupingSets(), ImmutableList.of(), node.getStep(), node.getHashSymbol(), node.getGroupIdSymbol()); }
private AggregationNode map(AggregationNode node, PlanNode source, PlanNodeId newNodeId) { ImmutableMap.Builder<Symbol, Aggregation> aggregations = ImmutableMap.builder(); for (Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { aggregations.put(map(entry.getKey()), map(entry.getValue())); } return new AggregationNode( newNodeId, source, aggregations.build(), groupingSets( mapAndDistinct(node.getGroupingKeys()), node.getGroupingSetCount(), node.getGlobalGroupingSets()), ImmutableList.of(), node.getStep(), node.getHashSymbol().map(this::map), node.getGroupIdSymbol().map(this::map)); }
@Override public PlanWithProperties visitAggregation(AggregationNode node, HashComputationSet parentPreference) { Optional<HashComputation> groupByHash = Optional.empty(); if (!node.isStreamable() && !canSkipHashGeneration(node.getGroupingKeys())) { groupByHash = computeHash(node.getGroupingKeys()); } // aggregation does not pass through preferred hash symbols HashComputationSet requiredHashes = new HashComputationSet(groupByHash); PlanWithProperties child = planAndEnforce(node.getSource(), requiredHashes, false, requiredHashes); Optional<Symbol> hashSymbol = groupByHash.map(child::getRequiredHashSymbol); return new PlanWithProperties( new AggregationNode( node.getId(), child.getNode(), node.getAggregations(), node.getGroupingSets(), node.getPreGroupedSymbols(), node.getStep(), hashSymbol, node.getGroupIdSymbol()), hashSymbol.isPresent() ? ImmutableMap.of(groupByHash.get(), hashSymbol.get()) : ImmutableMap.of()); }
private static boolean isSupportedAggregationNode(AggregationNode aggregationNode) { // Don't split streaming aggregations if (aggregationNode.isStreamable()) { return false; } if (aggregationNode.getHashSymbol().isPresent()) { // TODO: add support for hash symbol in aggregation node return false; } return aggregationNode.getStep() == PARTIAL && aggregationNode.getGroupingSetCount() == 1; }
private AggregationNode replaceAggregationSource( AggregationNode aggregation, PlanNode source, List<Symbol> groupingKeys) { return new AggregationNode( aggregation.getId(), source, aggregation.getAggregations(), singleGroupingSet(groupingKeys), ImmutableList.of(), aggregation.getStep(), aggregation.getHashSymbol(), aggregation.getGroupIdSymbol()); }
@Override public PlanNode replaceChildren(List<PlanNode> newChildren) { return new AggregationNode(getId(), Iterables.getOnlyElement(newChildren), aggregations, groupingSets, preGroupedSymbols, step, hashSymbol, groupIdSymbol); }
if (node.hasEmptyGroupingSet()) { return Optional.empty(); Optional<DecorrelationResult> childDecorrelationResultOptional = lookup.resolve(node.getSource()).accept(this, null); if (!childDecorrelationResultOptional.isPresent()) { return Optional.empty(); .map(node, childDecorrelationResult.node); Set<Symbol> groupingKeys = ImmutableSet.copyOf(node.getGroupingKeys()); List<Symbol> symbolsToAdd = childDecorrelationResult.symbolsToPropagate.stream() .filter(symbol -> !groupingKeys.contains(symbol)) decorrelatedAggregation.getId(), decorrelatedAggregation.getSource(), decorrelatedAggregation.getAggregations(), AggregationNode.singleGroupingSet(ImmutableList.<Symbol>builder() .addAll(node.getGroupingKeys()) .addAll(symbolsToAdd) .build()), ImmutableList.of(), decorrelatedAggregation.getStep(), decorrelatedAggregation.getHashSymbol(), decorrelatedAggregation.getGroupIdSymbol()); boolean atMostSingleRow = newAggregation.getGroupingSetCount() == 1 && constantSymbols.containsAll(newAggregation.getGroupingKeys());
AggregationNode aggregationNode = (AggregationNode) node; if (groupId.isPresent() != aggregationNode.getGroupIdSymbol().isPresent()) { return NO_MATCH; if (!matches(groupingSets.getGroupingKeys(), aggregationNode.getGroupingKeys(), symbolAliases)) { return NO_MATCH; if (groupingSets.getGroupingSetCount() != aggregationNode.getGroupingSetCount()) { return NO_MATCH; if (!groupingSets.getGlobalGroupingSets().equals(aggregationNode.getGlobalGroupingSets())) { return NO_MATCH; List<Symbol> aggregationsWithMask = aggregationNode.getAggregations() .entrySet() .stream() if (step != aggregationNode.getStep()) { return NO_MATCH; if (!matches(preGroupedSymbols, aggregationNode.getPreGroupedSymbols(), symbolAliases)) { return NO_MATCH;
private PlanNode addGatheringIntermediate(AggregationNode aggregation, PlanNodeIdAllocator idAllocator) { verify(aggregation.getGroupingKeys().isEmpty(), "Should be an un-grouped aggregation"); ExchangeNode gatheringExchange = ExchangeNode.gatheringExchange(idAllocator.getNextId(), ExchangeNode.Scope.LOCAL, aggregation); return new AggregationNode( idAllocator.getNextId(), gatheringExchange, outputsAsInputs(aggregation.getAggregations()), aggregation.getGroupingSets(), aggregation.getPreGroupedSymbols(), AggregationNode.Step.INTERMEDIATE, aggregation.getHashSymbol(), aggregation.getGroupIdSymbol()); }
@Override public Result apply(AggregationNode aggregation, Captures captures, Context context) { Lookup lookup = context.getLookup(); PlanNodeIdAllocator idAllocator = context.getIdAllocator(); Session session = context.getSession(); Optional<PlanNode> rewrittenSource = recurseToPartial(lookup.resolve(aggregation.getSource()), lookup, idAllocator); if (!rewrittenSource.isPresent()) { return Result.empty(); } PlanNode source = rewrittenSource.get(); if (getTaskConcurrency(session) > 1) { source = ExchangeNode.partitionedExchange( idAllocator.getNextId(), ExchangeNode.Scope.LOCAL, source, new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), source.getOutputSymbols())); source = new AggregationNode( idAllocator.getNextId(), source, inputsAsOutputs(aggregation.getAggregations()), aggregation.getGroupingSets(), aggregation.getPreGroupedSymbols(), AggregationNode.Step.INTERMEDIATE, aggregation.getHashSymbol(), aggregation.getGroupIdSymbol()); source = ExchangeNode.gatheringExchange(idAllocator.getNextId(), ExchangeNode.Scope.LOCAL, source); } return Result.ofPlanNode(aggregation.replaceChildren(ImmutableList.of(source))); }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(AggregationNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { if (node.getGroupingSetCount() != 1) { return Optional.empty(); } if (node.getStep() != SINGLE) { return Optional.empty(); } return Optional.of(groupBy( statsProvider.getStats(node.getSource()), node.getGroupingKeys(), node.getAggregations())); }
ImmutableList.Builder<Expression> nullLiterals = ImmutableList.builder(); ImmutableMap.Builder<Symbol, SymbolReference> sourcesSymbolMappingBuilder = ImmutableMap.builder(); for (Symbol sourceSymbol : referenceAggregation.getSource().getOutputSymbols()) { nullLiterals.add(nullLiteral); Symbol nullSymbol = symbolAllocator.newSymbol(nullLiteral, symbolAllocator.getTypes().get(sourceSymbol)); for (Map.Entry<Symbol, AggregationNode.Aggregation> entry : referenceAggregation.getAggregations().entrySet()) { Symbol aggregationSymbol = entry.getKey(); AggregationNode.Aggregation aggregation = entry.getValue(); AggregationNode aggregationOverNullRow = new AggregationNode( idAllocator.getNextId(), nullRow, aggregationsOverNullBuilder.build(), globalAggregation(), ImmutableList.of(), AggregationNode.Step.SINGLE,
@Test public void testGroupByEmpty() { PlanNode node = new AggregationNode( newId(), filter(baseTableScan, FALSE_LITERAL), ImmutableMap.of(), globalAggregation(), ImmutableList.of(), AggregationNode.Step.FINAL, Optional.empty(), Optional.empty()); Expression effectivePredicate = effectivePredicateExtractor.extract(node); assertEquals(effectivePredicate, TRUE_LITERAL); }
private PlanNode distinct(PlanNode node) { return new AggregationNode(idAllocator.getNextId(), node, ImmutableMap.of(), singleGroupingSet(node.getOutputSymbols()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty()); }
@Override public StreamProperties visitAggregation(AggregationNode node, List<StreamProperties> inputProperties) { StreamProperties properties = Iterables.getOnlyElement(inputProperties); // Only grouped symbols projected symbols are passed through return properties.translate(symbol -> node.getGroupingKeys().contains(symbol) ? Optional.of(symbol) : Optional.empty()); }
/** * Whether this node corresponds to a DISTINCT operation in SQL */ private static boolean isDistinct(AggregationNode node) { return node.getAggregations().isEmpty() && node.getOutputSymbols().size() == node.getGroupingKeys().size() && node.getOutputSymbols().containsAll(node.getGroupingKeys()); }
private PhysicalOperation planGlobalAggregation(AggregationNode node, PhysicalOperation source, LocalExecutionPlanContext context) { ImmutableMap.Builder<Symbol, Integer> outputMappings = ImmutableMap.builder(); AggregationOperatorFactory operatorFactory = createAggregationOperatorFactory( node.getId(), node.getAggregations(), node.getStep(), 0, outputMappings, source, context, node.getStep().isOutputPartial()); return new PhysicalOperation(operatorFactory, outputMappings.build(), context, source); }
@Override public Result apply(AggregationNode aggregationNode, Captures captures, Context context) { Set<Symbol> requiredInputs = Streams.concat( aggregationNode.getGroupingKeys().stream(), aggregationNode.getHashSymbol().map(Stream::of).orElse(Stream.empty()), aggregationNode.getAggregations().values().stream() .flatMap(PruneAggregationSourceColumns::getAggregationInputs)) .collect(toImmutableSet()); return restrictChildOutputs(context.getIdAllocator(), aggregationNode, requiredInputs) .map(Result::ofPlanNode) .orElse(Result.empty()); }
@Override public Map<Symbol, Symbol> visitAggregation(AggregationNode node, Set<Symbol> lookupSymbols) { Set<Symbol> groupByLookupSymbols = lookupSymbols.stream() .filter(node.getGroupingKeys()::contains) .collect(toImmutableSet()); checkState(!groupByLookupSymbols.isEmpty(), "No lookup symbols were able to pass through the aggregation group by"); return node.getSource().accept(this, groupByLookupSymbols); }
private static boolean isDistinctOperator(AggregationNode node) { return node.getAggregations().isEmpty(); } }