@Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, SymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { PlanWithProperties result = plan.accept(new Rewriter(idAllocator, symbolAllocator, session), PreferredProperties.any()); return result.getNode(); }
@Override public PlanWithProperties visitAggregation(AggregationNode node, PreferredProperties parentPreferredProperties) { Set<Symbol> partitioningRequirement = ImmutableSet.copyOf(node.getGroupingKeys()); boolean preferSingleNode = node.hasSingleNodeExecutionPreference(metadata.getFunctionRegistry()); PreferredProperties preferredProperties = preferSingleNode ? PreferredProperties.undistributed() : PreferredProperties.any(); if (!node.getGroupingKeys().isEmpty()) { preferredProperties = PreferredProperties.partitionedWithLocal(partitioningRequirement, grouped(node.getGroupingKeys())) .mergeWithParent(parentPreferredProperties); } PlanWithProperties child = planChild(node, preferredProperties); if (child.getProperties().isSingleNode()) { // If already unpartitioned, just drop the single aggregation back on return rebaseAndDeriveProperties(node, child); } if (preferSingleNode) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else if (!child.getProperties().isStreamPartitionedOn(partitioningRequirement) && !child.getProperties().isNodePartitionedOn(partitioningRequirement)) { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getGroupingKeys(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
private PlanWithProperties planPartitionedJoin(JoinNode node, List<Symbol> leftSymbols, List<Symbol> rightSymbols) { return planPartitionedJoin(node, leftSymbols, rightSymbols, node.getLeft().accept(this, PreferredProperties.partitioned(ImmutableSet.copyOf(leftSymbols)))); }
public PreferredProperties mergeWithParent(PreferredProperties parent) { List<LocalProperty<Symbol>> newLocal = ImmutableList.<LocalProperty<Symbol>>builder() .addAll(localProperties) .addAll(parent.getLocalProperties()) .build(); Builder builder = builder() .local(newLocal); if (globalProperties.isPresent()) { Global currentGlobal = globalProperties.get(); Global newGlobal = parent.getGlobalProperties() .map(currentGlobal::mergeWithParent) .orElse(currentGlobal); builder.global(newGlobal); } else { builder.global(parent.getGlobalProperties()); } return builder.build(); }
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, PreferredProperties preferredProperties) { List<Symbol> joinColumns = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getProbe) .collect(toImmutableList()); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = preferredProperties.getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(joinColumns), desiredLocalProperties) .mergeWithParent(preferredProperties)); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, PreferredProperties.any()); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, preferredProperties, probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
@Override public PlanWithProperties visitTopN(TopNNode node, PreferredProperties preferredProperties) { PlanWithProperties child; switch (node.getStep()) { case SINGLE: case FINAL: child = planChild(node, PreferredProperties.undistributed()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } break; case PARTIAL: child = planChild(node, PreferredProperties.any()); break; default: throw new UnsupportedOperationException(format("Unsupported step for TopN [%s]", node.getStep())); } return rebaseAndDeriveProperties(node, child); }
preferredChildProperties = PreferredProperties.any(); addExchange = partial -> gatheringExchange(idAllocator.getNextId(), REMOTE, partial); preferredChildProperties = PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getPartitionBy()), grouped(node.getPartitionBy())) .mergeWithParent(preferredProperties); addExchange = partial -> partitionedExchange(idAllocator.getNextId(), REMOTE, partial, node.getPartitionBy(), node.getHashSymbol());
PlanWithProperties child = planChild(node, PreferredProperties.undistributed()); PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getPartitionBy()), grouped(node.getPartitionBy())) .mergeWithParent(preferredProperties));
.addAll(parentProperties.getLocalProperties()) .build(); return hashPartitionedWithLocal(hashingColumns.get(), local); if (parentProperties.getGlobalProperties().isPresent()) { Global global = parentProperties.getGlobalProperties().get(); if (global.getPartitioningProperties().isPresent()) { Partitioning partitioning = global.getPartitioningProperties().get(); return hashPartitionedWithLocal(hashingSymbols, local); return partitionedWithLocal(common, local); return partitionedWithLocal(partitioningColumns, local); return partitionedWithLocal(partitioningColumns, local);
public static PreferredProperties any() { return builder().build(); }
@Override public PlanWithProperties visitUnion(UnionNode node, PreferredProperties parentPreference) Optional<PreferredProperties.Global> parentGlobal = parentPreference.getGlobalProperties(); if (parentGlobal.isPresent() && parentGlobal.get().isDistributed() && parentGlobal.get().getPartitioningProperties().isPresent()) { PreferredProperties.PartitioningProperties parentPartitioningPreference = parentGlobal.get().getPartitioningProperties().get(); Partitioning childPartitioning = desiredParentPartitioning.translate(createDirectTranslator(createMapping(node.getOutputSymbols(), node.sourceOutputLayout(sourceIndex)))); PreferredProperties childPreferred = PreferredProperties.builder() .global(PreferredProperties.Global.distributed(PreferredProperties.PartitioningProperties.partitioned(childPartitioning) .withNullsAndAnyReplicated(nullsAndAnyReplicated))) PlanWithProperties child = node.getSources().get(i).accept(this, PreferredProperties.any()); plannedChildren.add(child); if (child.getProperties().isSingleNode()) {
SetMultimap<Symbol, Symbol> filteringToSource = createMapping(filteringSourceSymbols, sourceSymbols); source = node.getSource().accept(this, PreferredProperties.partitioned(ImmutableSet.copyOf(sourceSymbols))); filteringSource = node.getFilteringSource().accept(this, PreferredProperties.partitionedWithNullsAndAnyReplicated(filteringPartitioning)); if (!source.getProperties().withReplicatedNulls(true).isCompatibleTablePartitioningWith(filteringSource.getProperties(), sourceToFiltering::get, metadata, session)) { filteringSource = withDerivedProperties( filteringSource = node.getFilteringSource().accept(this, PreferredProperties.partitionedWithNullsAndAnyReplicated(ImmutableSet.copyOf(filteringSourceSymbols))); source = node.getSource().accept(this, PreferredProperties.any()); filteringSource = node.getFilteringSource().accept(this, PreferredProperties.any());
@Override public PlanWithProperties visitOutput(OutputNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.undistributed()); if (!child.getProperties().isSingleNode() && isForceSingleNodeOutput(session)) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitWindow(WindowNode node, PreferredProperties preferredProperties) { List<LocalProperty<Symbol>> desiredProperties = new ArrayList<>(); if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } node.getOrderingScheme().ifPresent(orderingScheme -> orderingScheme.getOrderBy().stream() .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) .forEach(desiredProperties::add)); PlanWithProperties child = planChild( node, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getPartitionBy()), desiredProperties) .mergeWithParent(preferredProperties)); if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { if (node.getPartitionBy().isEmpty()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getPartitionBy(), node.getHashSymbol()), child.getProperties()); } } return rebaseAndDeriveProperties(node, child); }
? PreferredProperties.any() : PreferredProperties.derivePreferences(context.getPreferredProperties(), ImmutableSet.copyOf(node.getGroupBy()), Optional.of(node.getGroupBy()), grouped(node.getGroupBy()));
List<Symbol> filteringSourceSymbols = ImmutableList.of(node.getFilteringSourceJoinSymbol()); source = node.getSource().accept(this, context.withPreferredProperties(PreferredProperties.hashPartitioned(sourceSymbols))); filteringSource = node.getFilteringSource().accept(this, context.withPreferredProperties(PreferredProperties.any())); source = node.getSource().accept(this, context.withPreferredProperties(PreferredProperties.any())); filteringSource = node.getFilteringSource().accept(this, context.withPreferredProperties(PreferredProperties.any()).withHashPartitionedSemiJoinBanned(false));
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, Context context) { List<Symbol> joinColumns = Lists.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getProbe); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = context.getPreferredProperties().getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, context.withPreferredProperties(PreferredProperties.derivePreferences(context.getPreferredProperties(), ImmutableSet.copyOf(joinColumns), desiredLocalProperties))); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, context.withPreferredProperties(PreferredProperties.any())); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, context.getPreferredProperties(), probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
@Override public PlanWithProperties visitUnion(UnionNode node, Context context) if (!context.getPreferredProperties().getGlobalProperties().isPresent() || !context.getPreferredProperties().getGlobalProperties().get().isHashPartitioned()) { PlanWithProperties child = sources.get(i).accept(this, context.withPreferredProperties(PreferredProperties.any())); if (child.getProperties().isSingleNode()) { unpartitionedChildren.add(child.getNode()); List<Symbol> hashingColumns = context.getPreferredProperties().getGlobalProperties().get().getPartitioningProperties().get().getHashingOrder().get(); PlanWithProperties source = node.getSources().get(sourceIndex).accept(this, context.withPreferredProperties(PreferredProperties.hashPartitioned(sourceHashColumns))); if (!source.getProperties().isNodePartitionedOn(FIXED_HASH_DISTRIBUTION, sourceHashColumns)) { source = withDerivedProperties(
(PreferredProperties.local(ImmutableList.of(grouped("a", "b"))));
(PreferredProperties.partitionedWithLocal( ImmutableSet.of(symbol("a")), ImmutableList.of(grouped("a"))));