private static boolean groupsOnAllColumns(AggregationNode node, List<Symbol> columns) { return new HashSet<>(node.getGroupingKeys()).equals(new HashSet<>(columns)); }
private List<Symbol> getPushedDownGroupingSet(AggregationNode aggregation, Set<Symbol> availableSymbols, Set<Symbol> requiredJoinSymbols) { List<Symbol> groupingSet = aggregation.getGroupingKeys(); // keep symbols that are directly from the join's child (availableSymbols) List<Symbol> pushedDownGroupingSet = groupingSet.stream() .filter(availableSymbols::contains) .collect(Collectors.toList()); // add missing required join symbols to grouping set Set<Symbol> existingSymbols = new HashSet<>(pushedDownGroupingSet); requiredJoinSymbols.stream() .filter(existingSymbols::add) .forEach(pushedDownGroupingSet::add); return pushedDownGroupingSet; }
@Override public StreamProperties visitAggregation(AggregationNode node, List<StreamProperties> inputProperties) { StreamProperties properties = Iterables.getOnlyElement(inputProperties); // Only grouped symbols projected symbols are passed through return properties.translate(symbol -> node.getGroupingKeys().contains(symbol) ? Optional.of(symbol) : Optional.empty()); }
@Override public StreamProperties visitAggregation(AggregationNode node, List<StreamProperties> inputProperties) { StreamProperties properties = Iterables.getOnlyElement(inputProperties); // Only grouped symbols projected symbols are passed through return properties.translate(symbol -> node.getGroupingKeys().contains(symbol) ? Optional.of(symbol) : Optional.empty()); }
/** * Whether this node corresponds to a DISTINCT operation in SQL */ private static boolean isDistinct(AggregationNode node) { return node.getAggregations().isEmpty() && node.getOutputSymbols().size() == node.getGroupingKeys().size() && node.getOutputSymbols().containsAll(node.getGroupingKeys()); }
/** * Whether this node corresponds to a DISTINCT operation in SQL */ private static boolean isDistinct(AggregationNode node) { return node.getAggregations().isEmpty() && node.getOutputSymbols().size() == node.getGroupingKeys().size() && node.getOutputSymbols().containsAll(node.getGroupingKeys()); }
@Override public Map<Symbol, Symbol> visitAggregation(AggregationNode node, Set<Symbol> lookupSymbols) { Set<Symbol> groupByLookupSymbols = lookupSymbols.stream() .filter(node.getGroupingKeys()::contains) .collect(toImmutableSet()); checkState(!groupByLookupSymbols.isEmpty(), "No lookup symbols were able to pass through the aggregation group by"); return node.getSource().accept(this, groupByLookupSymbols); }
@Override public Map<Symbol, Symbol> visitAggregation(AggregationNode node, Set<Symbol> lookupSymbols) { Set<Symbol> groupByLookupSymbols = lookupSymbols.stream() .filter(node.getGroupingKeys()::contains) .collect(toImmutableSet()); checkState(!groupByLookupSymbols.isEmpty(), "No lookup symbols were able to pass through the aggregation group by"); return node.getSource().accept(this, groupByLookupSymbols); }
@Override public ActualProperties visitAggregation(AggregationNode node, List<ActualProperties> inputProperties) { ActualProperties properties = Iterables.getOnlyElement(inputProperties); ActualProperties translated = properties.translate(symbol -> node.getGroupingKeys().contains(symbol) ? Optional.of(symbol) : Optional.empty()); return ActualProperties.builderFrom(translated) .local(LocalProperties.grouped(node.getGroupingKeys())) .build(); }
@Override public ActualProperties visitAggregation(AggregationNode node, List<ActualProperties> inputProperties) { ActualProperties properties = Iterables.getOnlyElement(inputProperties); ActualProperties translated = properties.translate(symbol -> node.getGroupingKeys().contains(symbol) ? Optional.of(symbol) : Optional.empty()); return ActualProperties.builderFrom(translated) .local(LocalProperties.grouped(node.getGroupingKeys())) .build(); }
@Override public Result apply(AggregationNode aggregationNode, Captures captures, Context context) { Set<Symbol> requiredInputs = Streams.concat( aggregationNode.getGroupingKeys().stream(), aggregationNode.getHashSymbol().map(Stream::of).orElse(Stream.empty()), aggregationNode.getAggregations().values().stream() .flatMap(PruneAggregationSourceColumns::getAggregationInputs)) .collect(toImmutableSet()); return restrictChildOutputs(context.getIdAllocator(), aggregationNode, requiredInputs) .map(Result::ofPlanNode) .orElse(Result.empty()); }
@Override public Result apply(AggregationNode aggregationNode, Captures captures, Context context) { Set<Symbol> requiredInputs = Streams.concat( aggregationNode.getGroupingKeys().stream(), aggregationNode.getHashSymbol().map(Stream::of).orElse(Stream.empty()), aggregationNode.getAggregations().values().stream() .flatMap(PruneAggregationSourceColumns::getAggregationInputs)) .collect(toImmutableSet()); return restrictChildOutputs(context.getIdAllocator(), aggregationNode, requiredInputs) .map(Result::ofPlanNode) .orElse(Result.empty()); }
private static boolean isAggregationWithEmptyGroupBy(PlanNode planNode) { return searchFrom(planNode) .recurseOnlyWhen(MorePredicates.isInstanceOfAny(ProjectNode.class)) .where(AggregationNode.class::isInstance) .findFirst() .map(AggregationNode.class::cast) .map(aggregation -> aggregation.getGroupingKeys().isEmpty()) .orElse(false); }
private static boolean isAggregationWithEmptyGroupBy(PlanNode planNode) { return searchFrom(planNode) .recurseOnlyWhen(MorePredicates.isInstanceOfAny(ProjectNode.class)) .where(AggregationNode.class::isInstance) .findFirst() .map(AggregationNode.class::cast) .map(aggregation -> aggregation.getGroupingKeys().isEmpty()) .orElse(false); }
@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Context> context) { // Lookup symbols can only be passed through if they are part of the group by columns Set<Symbol> groupByLookupSymbols = context.get().getLookupSymbols().stream() .filter(node.getGroupingKeys()::contains) .collect(toImmutableSet()); if (groupByLookupSymbols.isEmpty()) { return node; } return context.defaultRewrite(node, new Context(groupByLookupSymbols, context.get().getSuccess())); }
@Override public PlanNode visitAggregation(AggregationNode node, RewriteContext<Context> context) { // Lookup symbols can only be passed through if they are part of the group by columns Set<Symbol> groupByLookupSymbols = context.get().getLookupSymbols().stream() .filter(node.getGroupingKeys()::contains) .collect(toImmutableSet()); if (groupByLookupSymbols.isEmpty()) { return node; } return context.defaultRewrite(node, new Context(groupByLookupSymbols, context.get().getSuccess())); }
@Override public Void visitAggregation(AggregationNode node, Integer indent) { output( indent, "%s aggregation over (%s)", node.getStep().name().toLowerCase(ENGLISH), node.getGroupingKeys().stream() .map(Object::toString) .sorted() .collect(joining(", "))); return visitPlan(node, indent + 1); }
@Override public PhysicalOperation visitAggregation(AggregationNode node, LocalExecutionPlanContext context) { PhysicalOperation source = node.getSource().accept(this, context); if (node.getGroupingKeys().isEmpty()) { return planGlobalAggregation(node, source, context); } boolean spillEnabled = isSpillEnabled(context.getSession()); DataSize unspillMemoryLimit = getAggregationOperatorUnspillMemoryLimit(context.getSession()); return planGroupByAggregation(node, source, spillEnabled, unspillMemoryLimit, context); }
@Override public PhysicalOperation visitAggregation(AggregationNode node, LocalExecutionPlanContext context) { PhysicalOperation source = node.getSource().accept(this, context); if (node.getGroupingKeys().isEmpty()) { return planGlobalAggregation(node, source, context); } boolean spillEnabled = isSpillEnabled(context.getSession()); DataSize unspillMemoryLimit = getAggregationOperatorUnspillMemoryLimit(context.getSession()); return planGroupByAggregation(node, source, spillEnabled, unspillMemoryLimit, context); }
private PlanNode addGatheringIntermediate(AggregationNode aggregation, PlanNodeIdAllocator idAllocator) { verify(aggregation.getGroupingKeys().isEmpty(), "Should be an un-grouped aggregation"); ExchangeNode gatheringExchange = ExchangeNode.gatheringExchange(idAllocator.getNextId(), ExchangeNode.Scope.LOCAL, aggregation); return new AggregationNode( idAllocator.getNextId(), gatheringExchange, outputsAsInputs(aggregation.getAggregations()), aggregation.getGroupingSets(), aggregation.getPreGroupedSymbols(), AggregationNode.Step.INTERMEDIATE, aggregation.getHashSymbol(), aggregation.getGroupIdSymbol()); }