for (int i = 0; i < node.getSources().size(); i++) { Map<Symbol, SymbolReference> outputsToInputs = new HashMap<>(); for (int index = 0; index < node.getInputs().get(i).size(); index++) { outputsToInputs.put( node.getOutputSymbols().get(index), node.getInputs().get(i).get(index).toSymbolReference()); PlanNode source = node.getSources().get(i); PlanNode rewrittenSource = context.rewrite(source, sourcePredicate); if (rewrittenSource != source) { return new ExchangeNode( node.getId(), node.getType(), node.getScope(), node.getPartitioningScheme(), builder.build(), node.getInputs(), node.getOrderingScheme());
public static ExchangeNode partitionedExchange(PlanNodeId id, Scope scope, PlanNode child, List<Symbol> partitioningColumns, Optional<Symbol> hashColumns) { return partitionedExchange(id, scope, child, partitioningColumns, hashColumns, false); }
@Override public PlanNode replaceChildren(List<PlanNode> newChildren) { return new ExchangeNode(getId(), type, scope, partitioningScheme, newChildren, inputs, orderingScheme); } }
public static Map<Symbol, Symbol> exchangeInputToOutput(ExchangeNode node, int sourceIndex) { List<Symbol> inputSymbols = node.getInputs().get(sourceIndex); Map<Symbol, Symbol> inputToOutput = new HashMap<>(); for (int i = 0; i < node.getOutputSymbols().size(); i++) { inputToOutput.put(inputSymbols.get(i), node.getOutputSymbols().get(i)); } return inputToOutput; }
@Override public Void visitExchange(ExchangeNode node, Set<Symbol> boundSymbols) { for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); checkDependencies(subplan.getOutputSymbols(), node.getInputs().get(i), "EXCHANGE subplan must provide all of the necessary symbols"); subplan.accept(this, boundSymbols); // visit child } checkDependencies(node.getOutputSymbols(), node.getPartitioningScheme().getOutputLayout(), "EXCHANGE must provide all of the necessary symbols for partition function"); return null; }
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
private PlanNode pushPartial(AggregationNode aggregation, ExchangeNode exchange, Context context) for (int i = 0; i < exchange.getSources().size(); i++) { PlanNode source = exchange.getSources().get(i); for (int outputIndex = 0; outputIndex < exchange.getOutputSymbols().size(); outputIndex++) { Symbol output = exchange.getOutputSymbols().get(outputIndex); Symbol input = exchange.getInputs().get(i).get(outputIndex); if (!output.equals(input)) { mappingsBuilder.put(output, input); exchange.getPartitioningScheme().getPartitioning(), aggregation.getOutputSymbols(), exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition()); return new ExchangeNode( context.getIdAllocator().getNextId(), exchange.getType(), exchange.getScope(), partitioning, partials,
@Override public Void visitExchange(ExchangeNode node, Integer indent) if (node.getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getOrderingScheme().get(); List<String> orderBy = orderingScheme.getOrderBy() .stream() UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, node.getScope().toString()), Joiner.on(", ").join(orderBy), formatOutputs(node.getOutputSymbols())); else if (node.getScope() == Scope.LOCAL) { print(indent, "- LocalExchange[%s%s]%s (%s) => %s", node.getPartitioningScheme().getPartitioning().getHandle(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), Joiner.on(", ").join(node.getPartitioningScheme().getPartitioning().getArguments()), formatOutputs(node.getOutputSymbols())); UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, node.getScope().toString()), node.getType(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), formatOutputs(node.getOutputSymbols())); printStats(indent + 2, node.getId());
@Override public StreamProperties visitExchange(ExchangeNode node, List<StreamProperties> inputProperties) { if (node.getOrderingScheme().isPresent()) { return StreamProperties.ordered(); } if (node.getScope() == REMOTE) { // TODO: correctly determine if stream is parallelised // based on session properties return StreamProperties.fixedStreams(); } switch (node.getType()) { case GATHER: return StreamProperties.singleStream(); case REPARTITION: if (node.getPartitioningScheme().getPartitioning().getHandle().equals(FIXED_ARBITRARY_DISTRIBUTION)) { return new StreamProperties(FIXED, Optional.empty(), false); } return new StreamProperties( FIXED, Optional.of(node.getPartitioningScheme().getPartitioning().getArguments().stream() .map(ArgumentBinding::getColumn) .collect(toImmutableList())), false); case REPLICATE: return new StreamProperties(MULTIPLE, Optional.empty(), false); } throw new UnsupportedOperationException("not yet implemented"); }
@Override public PhysicalOperation visitExchange(ExchangeNode node, LocalExecutionPlanContext context) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the local planner"); if (node.getOrderingScheme().isPresent()) { return createLocalMerge(node, context); } return createLocalExchange(node, context); }
@Override public Void visitExchange(ExchangeNode node, Integer indent) { Partitioning partitioning = node.getPartitioningScheme().getPartitioning(); output( indent, "%s exchange (%s, %s, %s)", node.getScope().name().toLowerCase(ENGLISH), node.getType(), partitioning.getHandle(), partitioning.getArguments().stream() .map(Object::toString) .sorted() // Currently, order of hash columns is not deterministic .collect(joining(", ", "[", "]"))); return visitPlan(node, indent + 1); }
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(ArgumentBinding::columnBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
@Override public Optional<SeenExchanges> visitExchange(ExchangeNode node, Void context) { Optional<SeenExchanges> seenExchangesOptional = aggregatedSeenExchanges(node.getSources()); if (!seenExchangesOptional.isPresent()) { // No partial aggregation below return Optional.empty(); } if (!node.getType().equals(REPARTITION)) { return seenExchangesOptional; } SeenExchanges seenExchanges = seenExchangesOptional.get(); if (node.getScope().equals(REMOTE)) { return Optional.of(new SeenExchanges(false, true)); } return Optional.of(new SeenExchanges(true, seenExchanges.remoteRepartitionExchange)); }
if (!right.getProperties().isSingleNode()) { right = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); replicatedExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); partitionedExchange(idAllocator.getNextId(), REMOTE, left.getNode(), ImmutableList.of(node.getLeftPartitionSymbol().get()), Optional.empty()), left.getProperties()); right = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, right.getNode(), ImmutableList.of(node.getRightPartitionSymbol().get()), Optional.empty()), right.getProperties());
@Override public Integer visitExchange(ExchangeNode node, Void context) { if (node.getScope() == ExchangeNode.Scope.REMOTE) { return 0; } return visitPlan(node, context); }
@Override public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType() == ExchangeNode.Type.GATHER) { return rebaseAndDeriveProperties(node, child); } // Always add an exchange because ExplainAnalyze should be in its own stage child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); return rebaseAndDeriveProperties(node, child); }
@Override public Set<PlanFragmentId> visitExchange(ExchangeNode node, PlanFragmentId currentFragmentId) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the phased execution scheduler"); ImmutableSet.Builder<PlanFragmentId> allSources = ImmutableSet.builder(); // Link the source fragments together, so we only schedule one at a time. Set<PlanFragmentId> previousSources = ImmutableSet.of(); for (PlanNode subPlanNode : node.getSources()) { Set<PlanFragmentId> currentSources = subPlanNode.accept(this, currentFragmentId); allSources.addAll(currentSources); addEdges(previousSources, currentSources); previousSources = currentSources; } return allSources.build(); }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(ExchangeNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { Optional<PlanNodeStatsEstimate> estimate = Optional.empty(); for (int i = 0; i < node.getSources().size(); i++) { PlanNode source = node.getSources().get(i); PlanNodeStatsEstimate sourceStats = statsProvider.getStats(source); PlanNodeStatsEstimate sourceStatsWithMappedSymbols = mapToOutputSymbols(sourceStats, node.getInputs().get(i), node.getOutputSymbols()); if (estimate.isPresent()) { estimate = Optional.of(addStatsAndMaxDistinctValues(estimate.get(), sourceStatsWithMappedSymbols)); } else { estimate = Optional.of(sourceStatsWithMappedSymbols); } } verify(estimate.isPresent()); return estimate; }
@Test public void testUnionOverSingleNodeAggregationAndUnion() { Plan plan = plan( "SELECT count(*) FROM (" + "SELECT 1 FROM nation GROUP BY regionkey " + "UNION ALL (" + " SELECT 1 FROM nation " + " UNION ALL " + " SELECT 1 FROM nation))", LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); List<PlanNode> remotes = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll(); assertEquals(remotes.size(), 2, "There should be exactly two RemoteExchanges"); assertEquals(((ExchangeNode) remotes.get(0)).getType(), GATHER); assertEquals(((ExchangeNode) remotes.get(1)).getType(), REPARTITION); }