@Override public PlanWithProperties visitJoin(JoinNode node, PreferredProperties preferredProperties) { List<Symbol> leftSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getLeft) .collect(toImmutableList()); List<Symbol> rightSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getRight) .collect(toImmutableList()); JoinNode.DistributionType distributionType = node.getDistributionType().orElseThrow(() -> new IllegalArgumentException("distributionType not yet set")); if (distributionType == JoinNode.DistributionType.REPLICATED) { PlanWithProperties left = node.getLeft().accept(this, PreferredProperties.any()); // use partitioned join if probe side is naturally partitioned on join symbols (e.g: because of aggregation) if (!node.getCriteria().isEmpty() && left.getProperties().isNodePartitionedOn(leftSymbols) && !left.getProperties().isSingleNode()) { return planPartitionedJoin(node, leftSymbols, rightSymbols, left); } return planReplicatedJoin(node, left); } else { return planPartitionedJoin(node, leftSymbols, rightSymbols); } }
@Override public PlanWithProperties visitTableWriter(TableWriterNode node, PreferredProperties preferredProperties) { PlanWithProperties source = node.getSource().accept(this, preferredProperties); Optional<PartitioningScheme> partitioningScheme = node.getPartitioningScheme(); if (!partitioningScheme.isPresent()) { if (scaleWriters) { partitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(SCALED_WRITER_DISTRIBUTION, ImmutableList.of()), source.getNode().getOutputSymbols())); } else if (redistributeWrites) { partitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), source.getNode().getOutputSymbols())); } } if (partitioningScheme.isPresent() && !source.getProperties().isCompatibleTablePartitioningWith(partitioningScheme.get().getPartitioning(), false, metadata, session)) { source = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), REMOTE, source.getNode(), partitioningScheme.get()), source.getProperties()); } return rebaseAndDeriveProperties(node, source); }
private PlanWithProperties rebaseAndDeriveProperties(PlanNode node, PlanWithProperties child) { return withDerivedProperties( ChildReplacer.replaceChildren(node, ImmutableList.of(child.getNode())), child.getProperties()); }
PlanWithProperties child = planChild(node, PreferredProperties.undistributed()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { child = withDerivedProperties( partitionedExchange( node.getPartitionBy(), node.getHashSymbol()), child.getProperties());
if (left.getProperties().isSingleNode()) { if (!right.getProperties().isSingleNode()) { right = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); right = withDerivedProperties( replicatedExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); left = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, left.getNode(), ImmutableList.of(node.getLeftPartitionSymbol().get()), Optional.empty()), left.getProperties()); right = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, right.getNode(), ImmutableList.of(node.getRightPartitionSymbol().get()), Optional.empty()), right.getProperties()); return new PlanWithProperties(newJoinNode, deriveProperties(newJoinNode, ImmutableList.of(left.getProperties(), right.getProperties())));
@Override public PlanWithProperties visitWindow(WindowNode node, PreferredProperties preferredProperties) { List<LocalProperty<Symbol>> desiredProperties = new ArrayList<>(); if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } node.getOrderingScheme().ifPresent(orderingScheme -> orderingScheme.getOrderBy().stream() .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) .forEach(desiredProperties::add)); PlanWithProperties child = planChild( node, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getPartitionBy()), desiredProperties) .mergeWithParent(preferredProperties)); if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { if (node.getPartitionBy().isEmpty()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getPartitionBy(), node.getHashSymbol()), child.getProperties()); } } return rebaseAndDeriveProperties(node, child); }
if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { true, node.getHashSymbol()), child.getProperties()); child = withDerivedProperties(addExchange.apply(child.getNode()), child.getProperties());
@Override public PlanWithProperties visitAggregation(AggregationNode node, PreferredProperties parentPreferredProperties) { Set<Symbol> partitioningRequirement = ImmutableSet.copyOf(node.getGroupingKeys()); boolean preferSingleNode = node.hasSingleNodeExecutionPreference(metadata.getFunctionRegistry()); PreferredProperties preferredProperties = preferSingleNode ? PreferredProperties.undistributed() : PreferredProperties.any(); if (!node.getGroupingKeys().isEmpty()) { preferredProperties = PreferredProperties.partitionedWithLocal(partitioningRequirement, grouped(node.getGroupingKeys())) .mergeWithParent(parentPreferredProperties); } PlanWithProperties child = planChild(node, preferredProperties); if (child.getProperties().isSingleNode()) { // If already unpartitioned, just drop the single aggregation back on return rebaseAndDeriveProperties(node, child); } if (preferSingleNode) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else if (!child.getProperties().isStreamPartitionedOn(partitioningRequirement) && !child.getProperties().isNodePartitionedOn(partitioningRequirement)) { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getGroupingKeys(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, PreferredProperties preferredProperties) { List<Symbol> joinColumns = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getProbe) .collect(toImmutableList()); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = preferredProperties.getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(joinColumns), desiredLocalProperties) .mergeWithParent(preferredProperties)); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, PreferredProperties.any()); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, preferredProperties, probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
private Partitioning selectUnionPartitioning(UnionNode node, PreferredProperties preferredProperties, PreferredProperties.PartitioningProperties parentPreference) { // Use the parent's requested partitioning if available if (parentPreference.getPartitioning().isPresent()) { return parentPreference.getPartitioning().get(); } // Try planning the children to see if any of them naturally produce a partitioning (for now, just select the first) boolean nullsAndAnyReplicated = parentPreference.isNullsAndAnyReplicated(); for (int sourceIndex = 0; sourceIndex < node.getSources().size(); sourceIndex++) { PreferredProperties.PartitioningProperties childPartitioning = parentPreference.translate(outputToInputTranslator(node, sourceIndex)).get(); PreferredProperties childPreferred = PreferredProperties.builder() .global(PreferredProperties.Global.distributed(childPartitioning.withNullsAndAnyReplicated(nullsAndAnyReplicated))) .build(); PlanWithProperties child = node.getSources().get(sourceIndex).accept(this, childPreferred); if (child.getProperties().isNodePartitionedOn(childPartitioning.getPartitioningColumns(), nullsAndAnyReplicated)) { Function<Symbol, Optional<Symbol>> childToParent = createTranslator(createMapping(node.sourceOutputLayout(sourceIndex), node.getOutputSymbols())); return child.getProperties().translate(childToParent).getNodePartitioning().get(); } } // Otherwise, choose an arbitrary partitioning over the columns return Partitioning.create(FIXED_HASH_DISTRIBUTION, ImmutableList.copyOf(parentPreference.getPartitioningColumns())); }
@Override public PlanWithProperties visitTopN(TopNNode node, PreferredProperties preferredProperties) { PlanWithProperties child; switch (node.getStep()) { case SINGLE: case FINAL: child = planChild(node, PreferredProperties.undistributed()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } break; case PARTIAL: child = planChild(node, PreferredProperties.any()); break; default: throw new UnsupportedOperationException(format("Unsupported step for TopN [%s]", node.getStep())); } return rebaseAndDeriveProperties(node, child); }
private PlanWithProperties planReplicatedJoin(JoinNode node, PlanWithProperties left) { // Broadcast Join PlanWithProperties right = node.getRight().accept(this, PreferredProperties.any()); if (left.getProperties().isSingleNode()) { if (!right.getProperties().isSingleNode() || (!isColocatedJoinEnabled(session) && hasMultipleSources(left.getNode(), right.getNode()))) { right = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); } } else { right = withDerivedProperties( replicatedExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); } return buildJoin(node, left, right, JoinNode.DistributionType.REPLICATED); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, PreferredProperties preferredProperties) { PreferredProperties preferredChildProperties = PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getDistinctSymbols()), grouped(node.getDistinctSymbols())) .mergeWithParent(preferredProperties); PlanWithProperties child = node.getSource().accept(this, preferredChildProperties); if (child.getProperties().isSingleNode() || !child.getProperties().isStreamPartitionedOn(node.getDistinctSymbols())) { child = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), REMOTE, child.getNode(), node.getDistinctSymbols(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitTableFinish(TableFinishNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { return rebaseAndDeriveProperties(node, child); } if (!child.getProperties().isCoordinatorOnly()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitDistinctLimit(DistinctLimitNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange( idAllocator.getNextId(), REMOTE, new DistinctLimitNode(idAllocator.getNextId(), child.getNode(), node.getLimit(), true, node.getDistinctSymbols(), node.getHashSymbol())), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
private PlanWithProperties buildJoin(JoinNode node, PlanWithProperties newLeft, PlanWithProperties newRight, JoinNode.DistributionType newDistributionType) { JoinNode result = new JoinNode(node.getId(), node.getType(), newLeft.getNode(), newRight.getNode(), node.getCriteria(), node.getOutputSymbols(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol(), Optional.of(newDistributionType)); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(newLeft.getProperties(), newRight.getProperties()))); }
@Override public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType() == ExchangeNode.Type.GATHER) { return rebaseAndDeriveProperties(node, child); } // Always add an exchange because ExplainAnalyze should be in its own stage child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitLimit(LimitNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( new LimitNode(idAllocator.getNextId(), child.getNode(), node.getCount(), true), child.getProperties()); child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitEnforceSingleRow(EnforceSingleRowNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitOutput(OutputNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.undistributed()); if (!child.getProperties().isSingleNode() && isForceSingleNodeOutput(session)) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }