private static int comparePartitionBy(WindowNode o1, WindowNode o2) { Iterator<Symbol> iterator1 = o1.getPartitionBy().iterator(); Iterator<Symbol> iterator2 = o2.getPartitionBy().iterator(); while (iterator1.hasNext() && iterator2.hasNext()) { Symbol symbol1 = iterator1.next(); Symbol symbol2 = iterator2.next(); int partitionByComparison = symbol1.compareTo(symbol2); if (partitionByComparison != 0) { return partitionByComparison; } } if (iterator1.hasNext()) { return 1; } if (iterator2.hasNext()) { return -1; } return 0; }
@Override public Map<Symbol, Symbol> visitWindow(WindowNode node, Set<Symbol> lookupSymbols) { Set<Symbol> partitionByLookupSymbols = lookupSymbols.stream() .filter(node.getPartitionBy()::contains) .collect(toImmutableSet()); checkState(!partitionByLookupSymbols.isEmpty(), "No lookup symbols were able to pass through the aggregation group by"); return node.getSource().accept(this, partitionByLookupSymbols); }
@Override public Void visitWindow(WindowNode node, Void context) { printNode(node, "Window", format("partition by = %s|order by = %s", Joiner.on(", ").join(node.getPartitionBy()), node.getOrderingScheme() .map(orderingScheme -> Joiner.on(", ").join(orderingScheme.getOrderBy())) .orElse("")), NODE_COLORS.get(NodeType.WINDOW)); return node.getSource().accept(this, context); }
public static boolean dependsOn(WindowNode parent, WindowNode child) { return parent.getPartitionBy().stream().anyMatch(child.getCreatedSymbols()::contains) || (parent.getOrderingScheme().isPresent() && parent.getOrderingScheme().get().getOrderBy().stream().anyMatch(child.getCreatedSymbols()::contains)) || parent.getWindowFunctions().values().stream() .map(WindowNode.Function::getFunctionCall) .map(SymbolsExtractor::extractUnique) .flatMap(Collection::stream) .anyMatch(child.getCreatedSymbols()::contains); } }
@Override public PlanNode visitWindow(WindowNode node, RewriteContext<Void> context) { checkState(node.getWindowFunctions().size() == 1, "WindowFilterPushdown requires that WindowNodes contain exactly one window function"); PlanNode rewrittenSource = context.rewrite(node.getSource()); if (canReplaceWithRowNumber(node)) { return new RowNumberNode(idAllocator.getNextId(), rewrittenSource, node.getPartitionBy(), getOnlyElement(node.getWindowFunctions().keySet()), Optional.empty(), Optional.empty()); } return replaceChildren(node, ImmutableList.of(rewrittenSource)); }
@Override public Void visitWindow(WindowNode node, Integer indent) List<String> partitionBy = Lists.transform(node.getPartitionBy(), Functions.toStringFunction()); List<Symbol> prePartitioned = node.getPartitionBy().stream() .filter(node.getPrePartitionedInputs()::contains) .collect(toImmutableList()); List<Symbol> notPrePartitioned = node.getPartitionBy().stream() .filter(column -> !node.getPrePartitionedInputs().contains(column)) .collect(toImmutableList());
@Override public PlanNode visitWindow(WindowNode node, RewriteContext<Expression> context) { List<Symbol> partitionSymbols = node.getPartitionBy(); // TODO: This could be broader. We can push down conjucts if they are constant for all rows in a window partition. // The simplest way to guarantee this is if the conjucts are deterministic functions of the partitioning symbols. // This can leave out cases where they're both functions of some set of common expressions and the partitioning // function is injective, but that's a rare case. The majority of window nodes are expected to be partitioned by // pre-projected symbols. Predicate<Expression> isSupported = conjunct -> DeterminismEvaluator.isDeterministic(conjunct) && SymbolsExtractor.extractUnique(conjunct).stream() .allMatch(partitionSymbols::contains); Map<Boolean, List<Expression>> conjuncts = extractConjuncts(context.get()).stream().collect(Collectors.partitioningBy(isSupported)); PlanNode rewrittenNode = context.defaultRewrite(node, combineConjuncts(conjuncts.get(true))); if (!conjuncts.get(false).isEmpty()) { rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, combineConjuncts(conjuncts.get(false))); } return rewrittenNode; }
.constrainTo(node.getSource().getOutputSymbols()) .withDefaultParallelism(session) .withPartitioning(node.getPartitionBy()); if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); if (!node.getPartitionBy().isEmpty()) { Optional<LocalProperty<Symbol>> groupingRequirement = matchIterator.next(); Set<Symbol> unPartitionedInputs = groupingRequirement.map(LocalProperty::getColumns).orElse(ImmutableSet.of()); prePartitionedInputs = node.getPartitionBy().stream() .filter(symbol -> !unPartitionedInputs.contains(symbol)) .collect(toImmutableSet()); if (prePartitionedInputs.equals(ImmutableSet.copyOf(node.getPartitionBy()))) { while (matchIterator.hasNext() && !matchIterator.next().isPresent()) { preSortedOrderPrefix++;
@Override public PlanWithProperties visitWindow(WindowNode node, PreferredProperties preferredProperties) { List<LocalProperty<Symbol>> desiredProperties = new ArrayList<>(); if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } node.getOrderingScheme().ifPresent(orderingScheme -> orderingScheme.getOrderBy().stream() .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) .forEach(desiredProperties::add)); PlanWithProperties child = planChild( node, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getPartitionBy()), desiredProperties) .mergeWithParent(preferredProperties)); if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { if (node.getPartitionBy().isEmpty()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getPartitionBy(), node.getHashSymbol()), child.getProperties()); } } return rebaseAndDeriveProperties(node, child); }
if (ImmutableSet.copyOf(node.getPartitionBy()).equals(node.getPrePartitionedInputs()) && (!orderingScheme.isPresent() || node.getPreSortedOrderPrefix() == orderingScheme.get().getOrderBy().size())) { return properties; if (!node.getPartitionBy().isEmpty()) { localProperties.add(new GroupingProperty<>(node.getPartitionBy()));
@Override public PlanNode visitWindow(WindowNode node, RewriteContext<Context> context) { if (!node.getWindowFunctions().values().stream() .map(function -> function.getFunctionCall().getName()) .allMatch(metadata.getFunctionRegistry()::isAggregationFunction)) { return node; } // Don't need this restriction if we can prove that all order by symbols are deterministically produced if (node.getOrderingScheme().isPresent()) { return node; } // Only RANGE frame type currently supported for aggregation functions because it guarantees the // same value for each peer group. // ROWS frame type requires the ordering to be fully deterministic (e.g. deterministically sorted on all columns) if (node.getFrames().stream().map(WindowNode.Frame::getType).anyMatch(type -> type != WindowFrame.Type.RANGE)) { // TODO: extract frames of type RANGE and allow optimization on them return node; } // Lookup symbols can only be passed through if they are part of the partitioning Set<Symbol> partitionByLookupSymbols = context.get().getLookupSymbols().stream() .filter(node.getPartitionBy()::contains) .collect(toImmutableSet()); if (partitionByLookupSymbols.isEmpty()) { return node; } return context.defaultRewrite(node, new Context(partitionByLookupSymbols, context.get().getSuccess())); }
.addAll(node.getPartitionBy());
.filter(referencedOutputs::contains) .iterator()) .addAll(windowNode.getPartitionBy());
@Override public Void visitWindow(WindowNode node, Set<Symbol> boundSymbols) { PlanNode source = node.getSource(); source.accept(this, boundSymbols); // visit child Set<Symbol> inputs = createInputs(source, boundSymbols); checkDependencies(inputs, node.getPartitionBy(), "Invalid node. Partition by symbols (%s) not in source plan output (%s)", node.getPartitionBy(), node.getSource().getOutputSymbols()); if (node.getOrderingScheme().isPresent()) { checkDependencies( inputs, node.getOrderingScheme().get().getOrderBy(), "Invalid node. Order by symbols (%s) not in source plan output (%s)", node.getOrderingScheme().get().getOrderBy(), node.getSource().getOutputSymbols()); } ImmutableList.Builder<Symbol> bounds = ImmutableList.builder(); for (WindowNode.Frame frame : node.getFrames()) { if (frame.getStartValue().isPresent()) { bounds.add(frame.getStartValue().get()); } if (frame.getEndValue().isPresent()) { bounds.add(frame.getEndValue().get()); } } checkDependencies(inputs, bounds.build(), "Invalid node. Frame bounds (%s) not in source plan output (%s)", bounds.build(), node.getSource().getOutputSymbols()); for (WindowNode.Function function : node.getWindowFunctions().values()) { Set<Symbol> dependencies = SymbolsExtractor.extractUnique(function.getFunctionCall()); checkDependencies(inputs, dependencies, "Invalid node. Window function dependencies (%s) not in source plan output (%s)", dependencies, node.getSource().getOutputSymbols()); } return null; }
@Override public PlanNode visitLimit(LimitNode node, RewriteContext<Void> context) { // Operators can handle MAX_VALUE rows per page, so do not optimize if count is greater than this value if (node.getCount() > Integer.MAX_VALUE) { return context.defaultRewrite(node); } PlanNode source = context.rewrite(node.getSource()); int limit = toIntExact(node.getCount()); if (source instanceof RowNumberNode) { RowNumberNode rowNumberNode = mergeLimit(((RowNumberNode) source), limit); if (rowNumberNode.getPartitionBy().isEmpty()) { return rowNumberNode; } source = rowNumberNode; } else if (source instanceof WindowNode && canOptimizeWindowFunction((WindowNode) source) && isOptimizeTopNRowNumber(session)) { WindowNode windowNode = (WindowNode) source; // verify that unordered row_number window functions are replaced by RowNumberNode verify(windowNode.getOrderingScheme().isPresent()); TopNRowNumberNode topNRowNumberNode = convertToTopNRowNumber(windowNode, limit); if (windowNode.getPartitionBy().isEmpty()) { return topNRowNumberNode; } source = topNRowNumberNode; } return replaceChildren(node, ImmutableList.of(source)); }
@Override public PlanWithProperties visitWindow(WindowNode node, HashComputationSet parentPreference) { if (node.getPartitionBy().isEmpty()) { return planSimpleNodeWithProperties(node, parentPreference, true); } Optional<HashComputation> hashComputation = computeHash(node.getPartitionBy()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), true, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); return new PlanWithProperties( new WindowNode( node.getId(), child.getNode(), node.getSpecification(), node.getWindowFunctions(), Optional.of(hashSymbol), node.getPrePartitionedInputs(), node.getPreSortedOrderPrefix()), child.getHashSymbols()); }
@Override public Map<Symbol, Symbol> visitWindow(WindowNode node, Set<Symbol> lookupSymbols) { Set<Symbol> partitionByLookupSymbols = lookupSymbols.stream() .filter(node.getPartitionBy()::contains) .collect(toImmutableSet()); checkState(!partitionByLookupSymbols.isEmpty(), "No lookup symbols were able to pass through the aggregation group by"); return node.getSource().accept(this, partitionByLookupSymbols); }
@Override public Void visitWindow(WindowNode node, Void context) { printNode(node, "Window", format("partition by = %s|order by = %s", Joiner.on(", ").join(node.getPartitionBy()), Joiner.on(", ").join(node.getOrderBy())), NODE_COLORS.get(NodeType.WINDOW)); return node.getSource().accept(this, context); }
private TopNRowNumberNode convertToTopNRowNumber(WindowNode windowNode, int limit) { return new TopNRowNumberNode(idAllocator.getNextId(), windowNode.getSource(), windowNode.getPartitionBy(), windowNode.getOrderBy(), windowNode.getOrderings(), getOnlyElement(windowNode.getWindowFunctions().keySet()), limit, false, Optional.empty()); }
@Override public PlanNode visitWindow(WindowNode node, RewriteContext<Void> context) { PlanNode rewrittenSource = context.rewrite(node.getSource()); if (canReplaceWithRowNumber(node)) { return new RowNumberNode(idAllocator.getNextId(), rewrittenSource, node.getPartitionBy(), getOnlyElement(node.getWindowFunctions().keySet()), Optional.empty(), Optional.empty()); } return replaceChildren(node, ImmutableList.of(rewrittenSource)); }