@Override public PlanNode replaceChildren(List<PlanNode> newChildren) { return new UnionNode(getId(), newChildren, getSymbolMapping(), getOutputSymbols()); } }
@Override public PlanNodeCostEstimate visitUnion(UnionNode node, Void context) { // this assumes that all union inputs will be gathered over the network // that is not aways true // but this estimate is better that returning UNKNOWN, as it sets // cumulative cost to unknown double inputSizeInBytes = getStats(node).getOutputSizeInBytes(node.getOutputSymbols(), types); return calculateRemoteGatherCost(inputSizeInBytes); }
@Override public Void visitUnion(UnionNode node, Integer indent) { print(indent, "- Union => [%s]", formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); return processChildren(node, indent + 1); }
@Override public Result apply(TopNNode topNNode, Captures captures, Context context) { UnionNode unionNode = captures.get(CHILD); ImmutableList.Builder<PlanNode> sources = ImmutableList.builder(); for (PlanNode source : unionNode.getSources()) { SymbolMapper.Builder symbolMapper = SymbolMapper.builder(); Set<Symbol> sourceOutputSymbols = ImmutableSet.copyOf(source.getOutputSymbols()); for (Symbol unionOutput : unionNode.getOutputSymbols()) { Set<Symbol> inputSymbols = ImmutableSet.copyOf(unionNode.getSymbolMapping().get(unionOutput)); Symbol unionInput = getLast(intersection(inputSymbols, sourceOutputSymbols)); symbolMapper.put(unionOutput, unionInput); } sources.add(symbolMapper.build().map(topNNode, source, context.getIdAllocator().getNextId())); } return Result.ofPlanNode(new UnionNode( unionNode.getId(), sources.build(), unionNode.getSymbolMapping(), unionNode.getOutputSymbols())); } }
@Override public PlanNode visitUnion(UnionNode node, RewriteContext<Expression> context) { boolean modified = false; ImmutableList.Builder<PlanNode> builder = ImmutableList.builder(); for (int i = 0; i < node.getSources().size(); i++) { Expression sourcePredicate = inlineSymbols(node.sourceSymbolMap(i), context.get()); PlanNode source = node.getSources().get(i); PlanNode rewrittenSource = context.rewrite(source, sourcePredicate); if (rewrittenSource != source) { modified = true; } builder.add(rewrittenSource); } if (modified) { return new UnionNode(node.getId(), builder.build(), node.getSymbolMapping(), node.getOutputSymbols()); } return node; }
@Override public PlanNode visitUnion(UnionNode node, RewriteContext<Void> context) { return new UnionNode(node.getId(), rewriteSources(node, context).build(), canonicalizeSetOperationSymbolMap(node.getSymbolMapping()), canonicalizeAndDistinct(node.getOutputSymbols())); }
HashComputationSet preference = parentPreference.pruneSymbols(node.getOutputSymbols()); for (Symbol outputSymbol : node.getOutputSymbols()) { outputToInputMap.put(outputSymbol, node.getSymbolMapping().get(outputSymbol).get(sourceId));
private PlanWithProperties arbitraryDistributeUnion( UnionNode node, List<PlanWithProperties> plannedChildren, List<PlanNode> partitionedChildren, List<List<Symbol>> partitionedOutputLayouts) { // TODO: can we insert LOCAL exchange for one child SOURCE distributed and another HASH distributed? if (countSources(partitionedChildren) == 0) { // No source distributed child, we can use insert LOCAL exchange // TODO: if all children have the same partitioning, pass this partitioning to the parent // instead of "arbitraryPartition". return new PlanWithProperties(node.replaceChildren( plannedChildren.stream() .map(PlanWithProperties::getNode) .collect(toList()))); } else { // Presto currently can not execute stage that has multiple table scans, so in that case // we have to insert REMOTE exchange with FIXED_ARBITRARY_DISTRIBUTION instead of local exchange return new PlanWithProperties( new ExchangeNode( idAllocator.getNextId(), REPARTITION, REMOTE, new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), node.getOutputSymbols()), partitionedChildren, partitionedOutputLayouts, Optional.empty())); } }
GATHER, LOCAL, new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), node.getOutputSymbols()), sources, inputLayouts, new PartitioningScheme( Partitioning.create(FIXED_HASH_DISTRIBUTION, preferredPartitionColumns.get()), node.getOutputSymbols(), Optional.empty()), sources, REPARTITION, LOCAL, new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), node.getOutputSymbols()), sources, inputLayouts,
@Override public PlanNode visitUnion(UnionNode node, RewriteContext<LimitContext> context) { LimitContext limit = context.get(); LimitContext childLimit = null; if (limit != null) { childLimit = new LimitContext(limit.getCount(), true); } List<PlanNode> sources = new ArrayList<>(); for (int i = 0; i < node.getSources().size(); i++) { sources.add(context.rewrite(node.getSources().get(i), childLimit)); } PlanNode output = new UnionNode(node.getId(), sources, node.getSymbolMapping(), node.getOutputSymbols()); if (limit != null) { output = new LimitNode(idAllocator.getNextId(), output, limit.getCount(), limit.isPartial()); } return output; }
Partitioning childPartitioning = desiredParentPartitioning.translate(createDirectTranslator(createMapping(node.getOutputSymbols(), node.sourceOutputLayout(sourceIndex)))); for (int column = 0; column < node.getOutputSymbols().size(); column++) { outputToSourcesMapping.put(node.getOutputSymbols().get(column), node.sourceOutputLayout(sourceIndex).get(column)); GATHER, REMOTE, new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), node.getOutputSymbols()), partitionedChildren, partitionedOutputLayouts, List<Symbol> exchangeOutputLayout = node.getOutputSymbols().stream() .map(outputSymbol -> symbolAllocator.newSymbol(outputSymbol.getName(), types.get(outputSymbol))) .collect(toImmutableList()); for (int i = 0; i < node.getOutputSymbols().size(); i++) { for (List<Symbol> outputLayout : unpartitionedOutputLayouts) { mappings.put(node.getOutputSymbols().get(i), outputLayout.get(i));
private Partitioning selectUnionPartitioning(UnionNode node, PreferredProperties preferredProperties, PreferredProperties.PartitioningProperties parentPreference) { // Use the parent's requested partitioning if available if (parentPreference.getPartitioning().isPresent()) { return parentPreference.getPartitioning().get(); } // Try planning the children to see if any of them naturally produce a partitioning (for now, just select the first) boolean nullsAndAnyReplicated = parentPreference.isNullsAndAnyReplicated(); for (int sourceIndex = 0; sourceIndex < node.getSources().size(); sourceIndex++) { PreferredProperties.PartitioningProperties childPartitioning = parentPreference.translate(outputToInputTranslator(node, sourceIndex)).get(); PreferredProperties childPreferred = PreferredProperties.builder() .global(PreferredProperties.Global.distributed(childPartitioning.withNullsAndAnyReplicated(nullsAndAnyReplicated))) .build(); PlanWithProperties child = node.getSources().get(sourceIndex).accept(this, childPreferred); if (child.getProperties().isNodePartitionedOn(childPartitioning.getPartitioningColumns(), nullsAndAnyReplicated)) { Function<Symbol, Optional<Symbol>> childToParent = createTranslator(createMapping(node.sourceOutputLayout(sourceIndex), node.getOutputSymbols())); return child.getProperties().translate(childToParent).getNodePartitioning().get(); } } // Otherwise, choose an arbitrary partitioning over the columns return Partitioning.create(FIXED_HASH_DISTRIBUTION, ImmutableList.copyOf(parentPreference.getPartitioningColumns())); }
public List<Symbol> sourceOutputLayout(int sourceIndex) { // Make sure the sourceOutputLayout symbols are listed in the same order as the corresponding output symbols return getOutputSymbols().stream() .map(symbol -> symbolMapping.get(symbol).get(sourceIndex)) .collect(toImmutableList()); }
@Override public Void visitUnion(UnionNode node, Void context) { for (PlanNode subPlanNode : node.getSources()) { subPlanNode.accept(this, context); } builder.addAll(node.getOutputSymbols()); return null; }
@Override public Void visitUnion(UnionNode node, Integer indent) { print(indent, "- Union => [%s]", formatOutputs(node.getOutputSymbols())); return processChildren(node, indent + 1); }
@Override public PlanNode visitUnion(UnionNode node, List<PlanNode> newChildren) { return new UnionNode(node.getId(), newChildren, node.getSymbolMapping(), node.getOutputSymbols()); }
@Override public PlanNode visitUnion(UnionNode node, RewriteContext<Void> context) { ImmutableList.Builder<PlanNode> rewrittenSources = ImmutableList.builder(); for (PlanNode source : node.getSources()) { rewrittenSources.add(context.rewrite(source)); } return new UnionNode(node.getId(), rewrittenSources.build(), canonicalizeUnionSymbolMap(node.getSymbolMapping()), canonicalize(node.getOutputSymbols())); }
/** * Returns the input to output symbol mapping for the given source channel. * A single input symbol can map to multiple output symbols, thus requiring a Multimap. */ public Multimap<Symbol, QualifiedNameReference> outputSymbolMap(int sourceIndex) { return Multimaps.transformValues(FluentIterable.from(getOutputSymbols()) .toMap(outputToSourceSymbolFunction(sourceIndex)) .asMultimap() .inverse(), Symbol::toQualifiedNameReference); }
@Override public PlanNode visitUnion(UnionNode node, RewriteContext<LimitContext> context) { LimitContext limit = context.get(); List<PlanNode> sources = new ArrayList<>(); for (int i = 0; i < node.getSources().size(); i++) { sources.add(context.rewrite(node.getSources().get(i), limit)); } PlanNode output = new UnionNode(node.getId(), sources, node.getSymbolMapping(), node.getOutputSymbols()); if (limit != null) { output = new LimitNode(idAllocator.getNextId(), output, limit.getCount()); } return output; }
@Override public PlanNode visitUnion(UnionNode node, RewriteContext<Expression> context) { boolean modified = false; ImmutableList.Builder<PlanNode> builder = ImmutableList.builder(); for (int i = 0; i < node.getSources().size(); i++) { Expression sourcePredicate = ExpressionTreeRewriter.rewriteWith(new ExpressionSymbolInliner(node.sourceSymbolMap(i)), context.get()); PlanNode source = node.getSources().get(i); PlanNode rewrittenSource = context.rewrite(source, sourcePredicate); if (rewrittenSource != source) { modified = true; } builder.add(rewrittenSource); } if (modified) { return new UnionNode(node.getId(), builder.build(), node.getSymbolMapping(), node.getOutputSymbols()); } return node; }