@Override protected Optional<PlanNode> manipulateAdjacentWindowNodes(WindowNode parent, WindowNode child, Context context) { if (!child.getSpecification().equals(parent.getSpecification()) || dependsOn(parent, child)) { return Optional.empty(); } ImmutableMap.Builder<Symbol, WindowNode.Function> functionsBuilder = ImmutableMap.builder(); functionsBuilder.putAll(parent.getWindowFunctions()); functionsBuilder.putAll(child.getWindowFunctions()); WindowNode mergedWindowNode = new WindowNode( parent.getId(), child.getSource(), parent.getSpecification(), functionsBuilder.build(), parent.getHashSymbol(), parent.getPrePartitionedInputs(), parent.getPreSortedOrderPrefix()); return Optional.of( restrictOutputs(context.getIdAllocator(), mergedWindowNode, ImmutableSet.copyOf(parent.getOutputSymbols())) .orElse(mergedWindowNode)); } }
@Override public Void visitWindow(WindowNode node, Set<Symbol> boundSymbols) { PlanNode source = node.getSource(); source.accept(this, boundSymbols); // visit child Set<Symbol> inputs = createInputs(source, boundSymbols); checkDependencies(inputs, node.getPartitionBy(), "Invalid node. Partition by symbols (%s) not in source plan output (%s)", node.getPartitionBy(), node.getSource().getOutputSymbols()); if (node.getOrderingScheme().isPresent()) { checkDependencies( inputs, node.getOrderingScheme().get().getOrderBy(), "Invalid node. Order by symbols (%s) not in source plan output (%s)", node.getOrderingScheme().get().getOrderBy(), node.getSource().getOutputSymbols()); } ImmutableList.Builder<Symbol> bounds = ImmutableList.builder(); for (WindowNode.Frame frame : node.getFrames()) { if (frame.getStartValue().isPresent()) { bounds.add(frame.getStartValue().get()); } if (frame.getEndValue().isPresent()) { bounds.add(frame.getEndValue().get()); } } checkDependencies(inputs, bounds.build(), "Invalid node. Frame bounds (%s) not in source plan output (%s)", bounds.build(), node.getSource().getOutputSymbols()); for (WindowNode.Function function : node.getWindowFunctions().values()) { Set<Symbol> dependencies = SymbolsExtractor.extractUnique(function.getFunctionCall()); checkDependencies(inputs, dependencies, "Invalid node. Window function dependencies (%s) not in source plan output (%s)", dependencies, node.getSource().getOutputSymbols()); } return null; }
@Override public PlanNode replaceChildren(List<PlanNode> newChildren) { return new WindowNode(getId(), Iterables.getOnlyElement(newChildren), specification, windowFunctions, hashSymbol, prePartitionedInputs, preSortedOrderPrefix); }
public static boolean dependsOn(WindowNode parent, WindowNode child) { return parent.getPartitionBy().stream().anyMatch(child.getCreatedSymbols()::contains) || (parent.getOrderingScheme().isPresent() && parent.getOrderingScheme().get().getOrderBy().stream().anyMatch(child.getCreatedSymbols()::contains)) || parent.getWindowFunctions().values().stream() .map(WindowNode.Function::getFunctionCall) .map(SymbolsExtractor::extractUnique) .flatMap(Collection::stream) .anyMatch(child.getCreatedSymbols()::contains); } }
private TopNRowNumberNode convertToTopNRowNumber(WindowNode windowNode, int limit) { return new TopNRowNumberNode(idAllocator.getNextId(), windowNode.getSource(), windowNode.getSpecification(), getOnlyElement(windowNode.getWindowFunctions().keySet()), limit, false, Optional.empty()); }
@Override public PlanNode visitWindow(WindowNode node, List<PlanNode> newChildren) { return new WindowNode( node.getId(), Iterables.getOnlyElement(newChildren), node.getPartitionBy(), node.getOrderBy(), node.getOrderings(), node.getFrame(), node.getWindowFunctions(), node.getSignatures(), node.getHashSymbol(), node.getPrePartitionedInputs(), node.getPreSortedOrderPrefix()); }
@Override public PlanWithProperties visitWindow(WindowNode node, HashComputationSet parentPreference) { if (node.getPartitionBy().isEmpty()) { return planSimpleNodeWithProperties(node, parentPreference, true); } Optional<HashComputation> hashComputation = computeHash(node.getPartitionBy()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), true, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); return new PlanWithProperties( new WindowNode( node.getId(), child.getNode(), node.getSpecification(), node.getWindowFunctions(), Optional.of(hashSymbol), node.getPrePartitionedInputs(), node.getPreSortedOrderPrefix()), child.getHashSymbols()); }
Optional<Symbol> hashSymbol = Optional.of(columnB); Set<Symbol> prePartitionedInputs = ImmutableSet.of(columnA); WindowNode windowNode = new WindowNode( id, sourceNode, assertEquals(actualNode.getId(), windowNode.getId()); assertEquals(actualNode.getSpecification(), windowNode.getSpecification()); assertEquals(actualNode.getWindowFunctions(), windowNode.getWindowFunctions()); assertEquals(actualNode.getFrames(), windowNode.getFrames()); assertEquals(actualNode.getHashSymbol(), windowNode.getHashSymbol()); assertEquals(actualNode.getPrePartitionedInputs(), windowNode.getPrePartitionedInputs()); assertEquals(actualNode.getPreSortedOrderPrefix(), windowNode.getPreSortedOrderPrefix());
.constrainTo(node.getSource().getOutputSymbols()) .withDefaultParallelism(session) .withPartitioning(node.getPartitionBy()); PlanWithProperties child = planAndEnforce(node.getSource(), childRequirements, childRequirements); if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); node.getOrderingScheme().ifPresent(orderingScheme -> orderingScheme.getOrderBy().stream() .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) if (!node.getPartitionBy().isEmpty()) { Optional<LocalProperty<Symbol>> groupingRequirement = matchIterator.next(); Set<Symbol> unPartitionedInputs = groupingRequirement.map(LocalProperty::getColumns).orElse(ImmutableSet.of()); prePartitionedInputs = node.getPartitionBy().stream() .filter(symbol -> !unPartitionedInputs.contains(symbol)) .collect(toImmutableSet()); if (prePartitionedInputs.equals(ImmutableSet.copyOf(node.getPartitionBy()))) { while (matchIterator.hasNext() && !matchIterator.next().isPresent()) { preSortedOrderPrefix++; WindowNode result = new WindowNode( node.getId(), child.getNode(), node.getSpecification(), node.getWindowFunctions(),
List<String> partitionBy = Lists.transform(node.getPartitionBy(), Functions.toStringFunction()); List<Symbol> prePartitioned = node.getPartitionBy().stream() .filter(node.getPrePartitionedInputs()::contains) .collect(toImmutableList()); List<Symbol> notPrePartitioned = node.getPartitionBy().stream() .filter(column -> !node.getPrePartitionedInputs().contains(column)) .collect(toImmutableList()); if (node.getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getOrderingScheme().get(); args.add(format("order by (%s)", Stream.concat( orderingScheme.getOrderBy().stream() .limit(node.getPreSortedOrderPrefix()) .map(symbol -> "<" + symbol + " " + orderingScheme.getOrdering(symbol) + ">"), orderingScheme.getOrderBy().stream() .skip(node.getPreSortedOrderPrefix()) .map(symbol -> symbol + " " + orderingScheme.getOrdering(symbol))) .collect(Collectors.joining(", ")))); print(indent, "- Window[%s]%s => [%s]", Joiner.on(", ").join(args), formatHash(node.getHashSymbol()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); for (Map.Entry<Symbol, WindowNode.Function> entry : node.getWindowFunctions().entrySet()) { FunctionCall call = entry.getValue().getFunctionCall(); String frameInfo = formatFrame(entry.getValue().getFrame());
@Override public Void visitWindow(WindowNode node, Integer indent) List<String> partitionBy = Lists.transform(node.getPartitionBy(), Functions.toStringFunction()); List<String> orderBy = Lists.transform(node.getOrderBy(), input -> input + " " + node.getOrderings().get(input)); List<Symbol> prePartitioned = node.getPartitionBy().stream() .filter(node.getPrePartitionedInputs()::contains) .collect(toImmutableList()); List<Symbol> notPrePartitioned = node.getPartitionBy().stream() .filter(column -> !node.getPrePartitionedInputs().contains(column)) .collect(toImmutableList()); node.getOrderBy().stream() .limit(node.getPreSortedOrderPrefix()) .map(symbol -> "<" + symbol + ">"), node.getOrderBy().stream() .skip(node.getPreSortedOrderPrefix()) .map(Symbol::toString)) .collect(Collectors.joining(", ")))); print(indent, "- Window[%s] => [%s]", Joiner.on(", ").join(args), formatOutputs(node.getOutputSymbols())); for (Map.Entry<Symbol, FunctionCall> entry : node.getWindowFunctions().entrySet()) { print(indent + 2, "%s := %s(%s)", entry.getKey(), entry.getValue().getName(), Joiner.on(", ").join(entry.getValue().getArguments()));
@Override public Void visitWindow(WindowNode node, Void context) { PlanNode source = node.getSource(); source.accept(this, context); // visit child verifyUniqueId(node); Set<Symbol> inputs = ImmutableSet.copyOf(source.getOutputSymbols()); checkDependencies(inputs, node.getPartitionBy(), "Invalid node. Partition by symbols (%s) not in source plan output (%s)", node.getPartitionBy(), node.getSource().getOutputSymbols()); checkDependencies(inputs, node.getOrderBy(), "Invalid node. Order by symbols (%s) not in source plan output (%s)", node.getOrderBy(), node.getSource().getOutputSymbols()); ImmutableList.Builder<Symbol> bounds = ImmutableList.builder(); if (node.getFrame().getStartValue().isPresent()) { bounds.add(node.getFrame().getStartValue().get()); } if (node.getFrame().getEndValue().isPresent()) { bounds.add(node.getFrame().getEndValue().get()); } checkDependencies(inputs, bounds.build(), "Invalid node. Frame bounds (%s) not in source plan output (%s)", bounds.build(), node.getSource().getOutputSymbols()); for (FunctionCall call : node.getWindowFunctions().values()) { Set<Symbol> dependencies = DependencyExtractor.extractUnique(call); checkDependencies(inputs, dependencies, "Invalid node. Window function dependencies (%s) not in source plan output (%s)", dependencies, node.getSource().getOutputSymbols()); } return null; }
public WindowNode window(WindowNode.Specification specification, Map<Symbol, WindowNode.Function> functions, PlanNode source) { return new WindowNode( idAllocator.getNextId(), source, specification, ImmutableMap.copyOf(functions), Optional.empty(), ImmutableSet.of(), 0); }
.map(alias -> alias.toSymbol(symbolAliases)) .collect(toImmutableSet()) .equals(windowNode.getPrePartitionedInputs())) .orElse(true)) { return NO_MATCH; .map(expectedSpecification -> expectedSpecification.getExpectedValue(symbolAliases) .equals(windowNode.getSpecification())) .orElse(true)) { return NO_MATCH; .map(Integer.valueOf(windowNode.getPreSortedOrderPrefix())::equals) .orElse(true)) { return NO_MATCH; .map(expectedHashSymbol -> expectedHashSymbol .map(alias -> alias.toSymbol(symbolAliases)) .equals(windowNode.getHashSymbol())) .orElse(true)) { return NO_MATCH;
Optional<OrderingScheme> orderingScheme = node.getOrderingScheme(); if (ImmutableSet.copyOf(node.getPartitionBy()).equals(node.getPrePartitionedInputs()) && (!orderingScheme.isPresent() || node.getPreSortedOrderPrefix() == orderingScheme.get().getOrderBy().size())) { return properties; if (!node.getPrePartitionedInputs().isEmpty()) { GroupingProperty<Symbol> prePartitionedProperty = new GroupingProperty<>(node.getPrePartitionedInputs()); for (LocalProperty<Symbol> localProperty : properties.getLocalProperties()) { if (!prePartitionedProperty.isSimplifiedBy(localProperty)) { if (!node.getPartitionBy().isEmpty()) { localProperties.add(new GroupingProperty<>(node.getPartitionBy()));
if (ImmutableSet.copyOf(node.getPartitionBy()).equals(node.getPrePartitionedInputs()) && node.getPreSortedOrderPrefix() == node.getOrderBy().size()) { return properties; if (!node.getPrePartitionedInputs().isEmpty()) { GroupingProperty<Symbol> prePartitionedProperty = new GroupingProperty<>(node.getPrePartitionedInputs()); for (LocalProperty<Symbol> localProperty : properties.getLocalProperties()) { if (!prePartitionedProperty.isSimplifiedBy(localProperty)) { if (!node.getPartitionBy().isEmpty()) { localProperties.add(new GroupingProperty<>(node.getPartitionBy())); for (Symbol column : node.getOrderBy()) { localProperties.add(new SortingProperty<>(column, node.getOrderings().get(column)));
@Override public PlanNode visitWindow(WindowNode node, RewriteContext<Void> context) { checkState(node.getWindowFunctions().size() == 1, "WindowFilterPushdown requires that WindowNodes contain exactly one window function"); PlanNode rewrittenSource = context.rewrite(node.getSource()); if (canReplaceWithRowNumber(node)) { return new RowNumberNode(idAllocator.getNextId(), rewrittenSource, node.getPartitionBy(), getOnlyElement(node.getWindowFunctions().keySet()), Optional.empty(), Optional.empty()); } return replaceChildren(node, ImmutableList.of(rewrittenSource)); }
private TopNRowNumberNode convertToTopNRowNumber(WindowNode windowNode, int limit) { return new TopNRowNumberNode(idAllocator.getNextId(), windowNode.getSource(), windowNode.getPartitionBy(), windowNode.getOrderBy(), windowNode.getOrderings(), getOnlyElement(windowNode.getWindowFunctions().keySet()), limit, false, Optional.empty()); }
@Override public PlanWithProperties visitWindow(WindowNode node, PreferredProperties preferredProperties) { List<LocalProperty<Symbol>> desiredProperties = new ArrayList<>(); if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } node.getOrderingScheme().ifPresent(orderingScheme -> orderingScheme.getOrderBy().stream() .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) .forEach(desiredProperties::add)); PlanWithProperties child = planChild( node, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getPartitionBy()), desiredProperties) .mergeWithParent(preferredProperties)); if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { if (node.getPartitionBy().isEmpty()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getPartitionBy(), node.getHashSymbol()), child.getProperties()); } } return rebaseAndDeriveProperties(node, child); }
private static boolean canOptimizeWindowFunction(WindowNode node) { if (node.getWindowFunctions().size() != 1) { return false; } Symbol rowNumberSymbol = getOnlyElement(node.getWindowFunctions().entrySet()).getKey(); return isRowNumberSignature(node.getWindowFunctions().get(rowNumberSymbol).getSignature()); }