@Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, SymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { PlanWithProperties result = plan.accept(new Rewriter(idAllocator, symbolAllocator, session), PreferredProperties.any()); return result.getNode(); }
@Override public PlanWithProperties visitJoin(JoinNode node, PreferredProperties preferredProperties) { List<Symbol> leftSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getLeft) .collect(toImmutableList()); List<Symbol> rightSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getRight) .collect(toImmutableList()); JoinNode.DistributionType distributionType = node.getDistributionType().orElseThrow(() -> new IllegalArgumentException("distributionType not yet set")); if (distributionType == JoinNode.DistributionType.REPLICATED) { PlanWithProperties left = node.getLeft().accept(this, PreferredProperties.any()); // use partitioned join if probe side is naturally partitioned on join symbols (e.g: because of aggregation) if (!node.getCriteria().isEmpty() && left.getProperties().isNodePartitionedOn(leftSymbols) && !left.getProperties().isSingleNode()) { return planPartitionedJoin(node, leftSymbols, rightSymbols, left); } return planReplicatedJoin(node, left); } else { return planPartitionedJoin(node, leftSymbols, rightSymbols); } }
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, PreferredProperties preferredProperties) { List<Symbol> joinColumns = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getProbe) .collect(toImmutableList()); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = preferredProperties.getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(joinColumns), desiredLocalProperties) .mergeWithParent(preferredProperties)); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, PreferredProperties.any()); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, preferredProperties, probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
@Override public PlanWithProperties visitEnforceSingleRow(EnforceSingleRowNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitTopN(TopNNode node, PreferredProperties preferredProperties) { PlanWithProperties child; switch (node.getStep()) { case SINGLE: case FINAL: child = planChild(node, PreferredProperties.undistributed()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } break; case PARTIAL: child = planChild(node, PreferredProperties.any()); break; default: throw new UnsupportedOperationException(format("Unsupported step for TopN [%s]", node.getStep())); } return rebaseAndDeriveProperties(node, child); }
PlanWithProperties child = node.getSources().get(i).accept(this, PreferredProperties.any()); plannedChildren.add(child); if (child.getProperties().isSingleNode()) {
@Override public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType() == ExchangeNode.Type.GATHER) { return rebaseAndDeriveProperties(node, child); } // Always add an exchange because ExplainAnalyze should be in its own stage child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitDistinctLimit(DistinctLimitNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange( idAllocator.getNextId(), REMOTE, new DistinctLimitNode(idAllocator.getNextId(), child.getNode(), node.getLimit(), true, node.getDistinctSymbols(), node.getHashSymbol())), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitTableFinish(TableFinishNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { return rebaseAndDeriveProperties(node, child); } if (!child.getProperties().isCoordinatorOnly()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitLimit(LimitNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( new LimitNode(idAllocator.getNextId(), child.getNode(), node.getCount(), true), child.getProperties()); child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Test public void testPickLayoutAnyPreference() Comparator<ActualProperties> preference = streamingExecutionPreference(PreferredProperties.any());
child = planChild(node, PreferredProperties.any());
preferredChildProperties = PreferredProperties.any(); addExchange = partial -> gatheringExchange(idAllocator.getNextId(), REMOTE, partial);
@Override public PlanWithProperties visitAggregation(AggregationNode node, PreferredProperties parentPreferredProperties) { Set<Symbol> partitioningRequirement = ImmutableSet.copyOf(node.getGroupingKeys()); boolean preferSingleNode = node.hasSingleNodeExecutionPreference(metadata.getFunctionRegistry()); PreferredProperties preferredProperties = preferSingleNode ? PreferredProperties.undistributed() : PreferredProperties.any(); if (!node.getGroupingKeys().isEmpty()) { preferredProperties = PreferredProperties.partitionedWithLocal(partitioningRequirement, grouped(node.getGroupingKeys())) .mergeWithParent(parentPreferredProperties); } PlanWithProperties child = planChild(node, preferredProperties); if (child.getProperties().isSingleNode()) { // If already unpartitioned, just drop the single aggregation back on return rebaseAndDeriveProperties(node, child); } if (preferSingleNode) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else if (!child.getProperties().isStreamPartitionedOn(partitioningRequirement) && !child.getProperties().isNodePartitionedOn(partitioningRequirement)) { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getGroupingKeys(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
private PlanWithProperties planReplicatedJoin(JoinNode node, PlanWithProperties left) { // Broadcast Join PlanWithProperties right = node.getRight().accept(this, PreferredProperties.any()); if (left.getProperties().isSingleNode()) { if (!right.getProperties().isSingleNode() || (!isColocatedJoinEnabled(session) && hasMultipleSources(left.getNode(), right.getNode()))) { right = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); } } else { right = withDerivedProperties( replicatedExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); } return buildJoin(node, left, right, JoinNode.DistributionType.REPLICATED); }
@Override public PlanNode optimize(PlanNode plan, Session session, Map<Symbol, Type> types, SymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator) { boolean distributedJoinEnabled = SystemSessionProperties.isDistributedJoinEnabled(session); boolean distributedIndexJoinEnabled = SystemSessionProperties.isDistributedIndexJoinEnabled(session); boolean redistributeWrites = SystemSessionProperties.isRedistributeWrites(session); boolean preferStreamingOperators = SystemSessionProperties.preferStreamingOperators(session); PlanWithProperties result = plan.accept(new Rewriter(symbolAllocator, idAllocator, symbolAllocator, session, distributedIndexJoinEnabled, distributedJoinEnabled, preferStreamingOperators, redistributeWrites), new Context(PreferredProperties.any(), false)); return result.getNode(); }
@Override public PlanWithProperties visitOutput(OutputNode node, Context context) { PlanWithProperties child = planChild(node, context.withPreferredProperties(PreferredProperties.any())); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitEnforceSingleRow(EnforceSingleRowNode node, Context context) { PlanWithProperties child = planChild(node, context.withPreferredProperties(PreferredProperties.any())); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }