@Override public Map<PlanNodeId, SplitSource> visitAggregation(AggregationNode node, Void context) { return node.getSource().accept(this, context); }
@Override public Map<Symbol, Symbol> visitAggregation(AggregationNode node, Set<Symbol> lookupSymbols) { Set<Symbol> groupByLookupSymbols = lookupSymbols.stream() .filter(node.getGroupingKeys()::contains) .collect(toImmutableSet()); checkState(!groupByLookupSymbols.isEmpty(), "No lookup symbols were able to pass through the aggregation group by"); return node.getSource().accept(this, groupByLookupSymbols); }
@Override public Void visitAggregation(AggregationNode node, Void context) { StringBuilder builder = new StringBuilder(); for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { if (entry.getValue().getMask().isPresent()) { builder.append(format("%s := %s (mask = %s)\\n", entry.getKey(), entry.getValue().getCall(), entry.getValue().getMask().get())); } else { builder.append(format("%s := %s\\n", entry.getKey(), entry.getValue().getCall())); } } printNode(node, format("Aggregate[%s]", node.getStep()), builder.toString(), NODE_COLORS.get(NodeType.AGGREGATE)); return node.getSource().accept(this, context); }
@Override public Void visitAggregation(AggregationNode node, Set<Symbol> boundSymbols) { PlanNode source = node.getSource(); source.accept(this, boundSymbols); // visit child Set<Symbol> inputs = createInputs(source, boundSymbols); checkDependencies(inputs, node.getGroupingKeys(), "Invalid node. Grouping key symbols (%s) not in source plan output (%s)", node.getGroupingKeys(), node.getSource().getOutputSymbols()); for (Aggregation aggregation : node.getAggregations().values()) { Set<Symbol> dependencies = SymbolsExtractor.extractUnique(aggregation.getCall()); checkDependencies(inputs, dependencies, "Invalid node. Aggregation dependencies (%s) not in source plan output (%s)", dependencies, node.getSource().getOutputSymbols()); aggregation.getMask().ifPresent(mask -> { checkDependencies(inputs, ImmutableSet.of(mask), "Invalid node. Aggregation mask symbol (%s) not in source plan output (%s)", mask, node.getSource().getOutputSymbols()); }); } return null; }
@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Void> context) { PlanNode source = context.rewrite(node.getSource()); //TODO: use mapper in other methods SymbolMapper mapper = new SymbolMapper(mapping); return mapper.map(node, source); }
@Override public Void visitAggregation(AggregationNode node, Void context) { if (node.getPreGroupedSymbols().isEmpty()) { return null; } StreamProperties properties = derivePropertiesRecursively(node.getSource(), metadata, sesstion, types, sqlParser); List<LocalProperty<Symbol>> desiredProperties = ImmutableList.of(new GroupingProperty<>(node.getPreGroupedSymbols())); Iterator<Optional<LocalProperty<Symbol>>> matchIterator = LocalProperties.match(properties.getLocalProperties(), desiredProperties).iterator(); Optional<LocalProperty<Symbol>> unsatisfiedRequirement = Iterators.getOnlyElement(matchIterator); checkArgument(!unsatisfiedRequirement.isPresent(), "Streaming aggregation with input not grouped on the grouping keys"); return null; } }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(AggregationNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { if (node.getGroupingSetCount() != 1) { return Optional.empty(); } if (node.getStep() != SINGLE) { return Optional.empty(); } return Optional.of(groupBy( statsProvider.getStats(node.getSource()), node.getGroupingKeys(), node.getAggregations())); }
@Override public Expression visitAggregation(AggregationNode node, Void context) { // GROUP BY () always produces a group, regardless of whether there's any // input (unlike the case where there are group by keys, which produce // no output if there's no input). // Therefore, we can't say anything about the effective predicate of the // output of such an aggregation. if (node.getGroupingKeys().isEmpty()) { return TRUE_LITERAL; } Expression underlyingPredicate = node.getSource().accept(this, context); return pullExpressionThroughSymbols(underlyingPredicate, node.getGroupingKeys()); }
@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getGroupingKeys()); if (node.getHashSymbol().isPresent()) { expectedInputs.add(node.getHashSymbol().get()); } ImmutableMap.Builder<Symbol, Aggregation> aggregations = ImmutableMap.builder(); for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { Symbol symbol = entry.getKey(); if (context.get().contains(symbol)) { Aggregation aggregation = entry.getValue(); expectedInputs.addAll(SymbolsExtractor.extractUnique(aggregation.getCall())); aggregation.getMask().ifPresent(expectedInputs::add); aggregations.put(symbol, aggregation); } } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new AggregationNode(node.getId(), source, aggregations.build(), node.getGroupingSets(), ImmutableList.of(), node.getStep(), node.getHashSymbol(), node.getGroupIdSymbol()); }
@Override protected Optional<PlanNode> pushDownProjectOff( PlanNodeIdAllocator idAllocator, AggregationNode aggregationNode, Set<Symbol> referencedOutputs) { Map<Symbol, AggregationNode.Aggregation> prunedAggregations = Maps.filterKeys( aggregationNode.getAggregations(), referencedOutputs::contains); if (prunedAggregations.size() == aggregationNode.getAggregations().size()) { return Optional.empty(); } // PruneAggregationSourceColumns will subsequently project off any newly unused inputs. return Optional.of( new AggregationNode( aggregationNode.getId(), aggregationNode.getSource(), prunedAggregations, aggregationNode.getGroupingSets(), aggregationNode.getPreGroupedSymbols(), aggregationNode.getStep(), aggregationNode.getHashSymbol(), aggregationNode.getGroupIdSymbol())); } }
@Override public PlanNodeCostEstimate visitAggregation(AggregationNode node, Void context) { if (node.getStep() != FINAL && node.getStep() != SINGLE) { return PlanNodeCostEstimate.unknown(); } PlanNodeStatsEstimate aggregationStats = getStats(node); PlanNodeStatsEstimate sourceStats = getStats(node.getSource()); double cpuCost = sourceStats.getOutputSizeInBytes(node.getSource().getOutputSymbols(), types); double memoryCost = aggregationStats.getOutputSizeInBytes(node.getOutputSymbols(), types); return new PlanNodeCostEstimate(cpuCost, memoryCost, 0); }
@Override public PhysicalOperation visitAggregation(AggregationNode node, LocalExecutionPlanContext context) { PhysicalOperation source = node.getSource().accept(this, context); if (node.getGroupingKeys().isEmpty()) { return planGlobalAggregation(node, source, context); } boolean spillEnabled = isSpillEnabled(context.getSession()); DataSize unspillMemoryLimit = getAggregationOperatorUnspillMemoryLimit(context.getSession()); return planGroupByAggregation(node, source, spillEnabled, unspillMemoryLimit, context); }
@Override public Result apply(AggregationNode parent, Captures captures, Context context) { if (!parent.hasDefaultOutput() || parent.getOutputSymbols().size() != 1) { return Result.empty(); } Map<Symbol, AggregationNode.Aggregation> assignments = parent.getAggregations(); for (Map.Entry<Symbol, AggregationNode.Aggregation> entry : assignments.entrySet()) { AggregationNode.Aggregation aggregation = entry.getValue(); requireNonNull(aggregation, "aggregation is null"); Signature signature = aggregation.getSignature(); FunctionCall functionCall = aggregation.getCall(); if (!"count".equals(signature.getName()) || !functionCall.getArguments().isEmpty()) { return Result.empty(); } } if (!assignments.isEmpty() && isScalar(parent.getSource(), context.getLookup())) { return Result.ofPlanNode(new ValuesNode(parent.getId(), parent.getOutputSymbols(), ImmutableList.of(ImmutableList.of(new LongLiteral("1"))))); } return Result.empty(); } }
@Override public PlanWithProperties visitAggregation(AggregationNode node, HashComputationSet parentPreference) { Optional<HashComputation> groupByHash = Optional.empty(); if (!node.isStreamable() && !canSkipHashGeneration(node.getGroupingKeys())) { groupByHash = computeHash(node.getGroupingKeys()); } // aggregation does not pass through preferred hash symbols HashComputationSet requiredHashes = new HashComputationSet(groupByHash); PlanWithProperties child = planAndEnforce(node.getSource(), requiredHashes, false, requiredHashes); Optional<Symbol> hashSymbol = groupByHash.map(child::getRequiredHashSymbol); return new PlanWithProperties( new AggregationNode( node.getId(), child.getNode(), node.getAggregations(), node.getGroupingSets(), node.getPreGroupedSymbols(), node.getStep(), hashSymbol, node.getGroupIdSymbol()), hashSymbol.isPresent() ? ImmutableMap.of(groupByHash.get(), hashSymbol.get()) : ImmutableMap.of()); }
@Override public PlanNodeCostEstimate visitAggregation(AggregationNode node, Void context) { PlanNode source = node.getSource(); double inputSizeInBytes = getStats(source).getOutputSizeInBytes(source.getOutputSymbols(), types); PlanNodeCostEstimate remoteRepartitionCost = calculateRemoteRepartitionCost(inputSizeInBytes); PlanNodeCostEstimate localRepartitionCost = calculateLocalRepartitionCost(inputSizeInBytes); // TODO consider cost of aggregation itself, not only exchanges, based on aggregation's properties return remoteRepartitionCost.add(localRepartitionCost); }
@Override public GroupedExecutionProperties visitAggregation(AggregationNode node, Void context) { GroupedExecutionProperties properties = node.getSource().accept(this, null); if (groupedExecutionForAggregation && properties.isCurrentNodeCapable()) { switch (node.getStep()) { case SINGLE: case FINAL: return new GroupedExecutionProperties(true, true, properties.capableTableScanNodes); case PARTIAL: case INTERMEDIATE: return properties; } } return GroupedExecutionProperties.notCapable(); }
@Override @Deprecated public PlanNode visitAggregation(AggregationNode node, RewriteContext<LimitContext> context) { LimitContext limit = context.get(); if (limit != null && node.getAggregations().isEmpty() && node.getOutputSymbols().size() == node.getGroupingKeys().size() && node.getOutputSymbols().containsAll(node.getGroupingKeys())) { PlanNode rewrittenSource = context.rewrite(node.getSource()); return new DistinctLimitNode(idAllocator.getNextId(), rewrittenSource, limit.getCount(), false, rewrittenSource.getOutputSymbols(), Optional.empty()); } PlanNode rewrittenNode = context.defaultRewrite(node); if (limit != null) { // Drop in a LimitNode b/c limits cannot be pushed through aggregations rewrittenNode = new LimitNode(idAllocator.getNextId(), rewrittenNode, limit.getCount(), limit.isPartial()); } return rewrittenNode; }
@Override public Result apply(LimitNode parent, Captures captures, Context context) { AggregationNode child = captures.get(CHILD); return Result.ofPlanNode( new DistinctLimitNode( parent.getId(), child.getSource(), parent.getCount(), false, child.getGroupingKeys(), child.getHashSymbol())); } }
public PlanNode rewriteScalarAggregation(LateralJoinNode lateralJoinNode, AggregationNode aggregation) { List<Symbol> correlation = lateralJoinNode.getCorrelation(); Optional<DecorrelatedNode> source = planNodeDecorrelator.decorrelateFilters(lookup.resolve(aggregation.getSource()), correlation); if (!source.isPresent()) { return lateralJoinNode; } Symbol nonNull = symbolAllocator.newSymbol("non_null", BooleanType.BOOLEAN); Assignments scalarAggregationSourceAssignments = Assignments.builder() .putIdentities(source.get().getNode().getOutputSymbols()) .put(nonNull, TRUE_LITERAL) .build(); ProjectNode scalarAggregationSourceWithNonNullableSymbol = new ProjectNode( idAllocator.getNextId(), source.get().getNode(), scalarAggregationSourceAssignments); return rewriteScalarAggregation( lateralJoinNode, aggregation, scalarAggregationSourceWithNonNullableSymbol, source.get().getCorrelatedPredicates(), nonNull); }
@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Boolean> context) { boolean distinct = isDistinctOperator(node); PlanNode rewrittenNode = context.rewrite(node.getSource(), distinct); if (context.get() && distinct) { // Assumes underlying node has same output symbols as this distinct node return rewrittenNode; } return new AggregationNode( node.getId(), rewrittenNode, node.getAggregations(), node.getGroupingSets(), ImmutableList.of(), node.getStep(), node.getHashSymbol(), node.getGroupIdSymbol()); }