@Override public PlanNode replaceChildren(List<PlanNode> newChildren) { return new UnionNode(getId(), newChildren, getSymbolMapping(), getOutputSymbols()); } }
@Override public Result apply(TableWriterNode writerNode, Captures captures, Context context) { UnionNode unionNode = captures.get(CHILD); ImmutableList.Builder<PlanNode> rewrittenSources = ImmutableList.builder(); List<Map<Symbol, Symbol>> sourceMappings = new ArrayList<>(); for (int source = 0; source < unionNode.getSources().size(); source++) { rewrittenSources.add(rewriteSource(writerNode, unionNode, source, sourceMappings, context)); } ImmutableListMultimap.Builder<Symbol, Symbol> unionMappings = ImmutableListMultimap.builder(); sourceMappings.forEach(mappings -> mappings.forEach(unionMappings::put)); return Result.ofPlanNode( new UnionNode( context.getIdAllocator().getNextId(), rewrittenSources.build(), unionMappings.build(), ImmutableList.copyOf(unionMappings.build().keySet()))); }
@Override public Expression visitUnion(UnionNode node, Void context) { return deriveCommonPredicates(node, source -> node.outputSymbolMap(source).entries()); }
@Override protected final Optional<PlanNodeStatsEstimate> doCalculate(UnionNode node, StatsProvider statsProvider, Lookup lookup, Session session, TypeProvider types) { checkArgument(!node.getSources().isEmpty(), "Empty Union is not supported"); Optional<PlanNodeStatsEstimate> estimate = Optional.empty(); for (int i = 0; i < node.getSources().size(); i++) { PlanNode source = node.getSources().get(i); PlanNodeStatsEstimate sourceStats = statsProvider.getStats(source); PlanNodeStatsEstimate sourceStatsWithMappedSymbols = mapToOutputSymbols(sourceStats, node.getSymbolMapping(), i); if (estimate.isPresent()) { estimate = Optional.of(addStatsAndCollapseDistinctValues(estimate.get(), sourceStatsWithMappedSymbols)); } else { estimate = Optional.of(sourceStatsWithMappedSymbols); } } return estimate; }
@Override public PlanNode visitUnion(UnionNode node, RewriteContext<Set<Symbol>> context) { ListMultimap<Symbol, Symbol> rewrittenSymbolMapping = rewriteSetOperationSymbolMapping(node, context); ImmutableList<PlanNode> rewrittenSubPlans = rewriteSetOperationSubPlans(node, context, rewrittenSymbolMapping); return new UnionNode(node.getId(), rewrittenSubPlans, rewrittenSymbolMapping, ImmutableList.copyOf(rewrittenSymbolMapping.keySet())); }
private UnionNode union(List<PlanNode> nodes, List<Symbol> outputs) { ImmutableListMultimap.Builder<Symbol, Symbol> outputsToInputs = ImmutableListMultimap.builder(); for (PlanNode source : nodes) { for (int i = 0; i < source.getOutputSymbols().size(); i++) { outputsToInputs.put(outputs.get(i), source.getOutputSymbols().get(i)); } } return new UnionNode(idAllocator.getNextId(), nodes, outputsToInputs.build(), outputs); }
for (int i = 0; i < source.getSources().size(); i++) { Map<Symbol, SymbolReference> outputToInput = source.sourceSymbolMap(i); // Map: output of union -> input of this source to the union projectSymbolMapping.put(entry.getKey(), symbol); outputSources.add(new ProjectNode(context.getIdAllocator().getNextId(), source.getSources().get(i), assignments.build())); outputLayout.forEach(symbol -> mappings.put(symbol, projectSymbolMapping.get(symbol))); return Result.ofPlanNode(new UnionNode(parent.getId(), outputSources.build(), mappings.build(), ImmutableList.copyOf(mappings.build().keySet())));
private Partitioning selectUnionPartitioning(UnionNode node, PreferredProperties preferredProperties, PreferredProperties.PartitioningProperties parentPreference) { // Use the parent's requested partitioning if available if (parentPreference.getPartitioning().isPresent()) { return parentPreference.getPartitioning().get(); } // Try planning the children to see if any of them naturally produce a partitioning (for now, just select the first) boolean nullsAndAnyReplicated = parentPreference.isNullsAndAnyReplicated(); for (int sourceIndex = 0; sourceIndex < node.getSources().size(); sourceIndex++) { PreferredProperties.PartitioningProperties childPartitioning = parentPreference.translate(outputToInputTranslator(node, sourceIndex)).get(); PreferredProperties childPreferred = PreferredProperties.builder() .global(PreferredProperties.Global.distributed(childPartitioning.withNullsAndAnyReplicated(nullsAndAnyReplicated))) .build(); PlanWithProperties child = node.getSources().get(sourceIndex).accept(this, childPreferred); if (child.getProperties().isNodePartitionedOn(childPartitioning.getPartitioningColumns(), nullsAndAnyReplicated)) { Function<Symbol, Optional<Symbol>> childToParent = createTranslator(createMapping(node.sourceOutputLayout(sourceIndex), node.getOutputSymbols())); return child.getProperties().translate(childToParent).getNodePartitioning().get(); } } // Otherwise, choose an arbitrary partitioning over the columns return Partitioning.create(FIXED_HASH_DISTRIBUTION, ImmutableList.copyOf(parentPreference.getPartitioningColumns())); }
ImmutableList.Builder<PlanNode> rewrittenSources = ImmutableList.builder(); ImmutableListMultimap.Builder<Symbol, Symbol> mappings = ImmutableListMultimap.builder(); for (int i = 0; i < unionNode.getSources().size(); i++) { PlanNode unionOriginalSource = unionNode.getSources().get(i); ImmutableList.Builder<Symbol> newSymbols = ImmutableList.builder(); for (Symbol outputSymbol : node.getOutputSymbols()) { unionOriginalSource, node.getTarget(), unionNode.sourceOutputLayout(i), node.getColumnNames(), newSymbols.build(), return new UnionNode(idAllocator.getNextId(), rewrittenSources.build(), mappings.build(), ImmutableList.copyOf(mappings.build().keySet()));
@Override public Void visitUnion(UnionNode node, Void context) { for (PlanNode subPlanNode : node.getSources()) { subPlanNode.accept(this, context); } return null; }
InMemoryExchange inMemoryExchange = new InMemoryExchange(types); for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); List<Symbol> expectedLayout = node.sourceOutputLayout(i); OperatorFactory operatorFactory = new FilterAndProjectOperator.FilterAndProjectOperatorFactory( subContext.getNextOperatorId(), node.getId(), compiler.compilePageProcessor(trueExpression(), projections), projections.stream() operatorFactories.add(inMemoryExchange.createSinkFactory(subContext.getNextOperatorId(), node.getId())); return new PhysicalOperation(createRandomDistribution(context.getNextOperatorId(), node.getId(), inMemoryExchange), makeLayout(node));
@Override public Void visitUnion(UnionNode node, Integer indent) { print(indent, "- Union => [%s]", formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); return processChildren(node, indent + 1); }
@Override public Void visitUnion(UnionNode node, Void context) { for (PlanNode subPlanNode : node.getSources()) { subPlanNode.accept(this, context); } builder.addAll(node.getOutputSymbols()); return null; }
@Override public Void visitUnion(UnionNode node, Void context) { for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); checkDependencies(subplan.getOutputSymbols(), node.sourceOutputLayout(i), "UNION subplan must provide all of the necessary symbols"); subplan.accept(this, context); // visit child } verifyUniqueId(node); return null; }
private PlanWithProperties arbitraryDistributeUnion( UnionNode node, List<PlanWithProperties> plannedChildren, List<PlanNode> partitionedChildren, List<List<Symbol>> partitionedOutputLayouts) { // TODO: can we insert LOCAL exchange for one child SOURCE distributed and another HASH distributed? if (countSources(partitionedChildren) == 0) { // No source distributed child, we can use insert LOCAL exchange // TODO: if all children have the same partitioning, pass this partitioning to the parent // instead of "arbitraryPartition". return new PlanWithProperties(node.replaceChildren( plannedChildren.stream() .map(PlanWithProperties::getNode) .collect(toList()))); } else { // Presto currently can not execute stage that has multiple table scans, so in that case // we have to insert REMOTE exchange with FIXED_ARBITRARY_DISTRIBUTION instead of local exchange return new PlanWithProperties( new ExchangeNode( idAllocator.getNextId(), REPARTITION, REMOTE, new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), node.getOutputSymbols()), partitionedChildren, partitionedOutputLayouts, Optional.empty())); } }
private Function<Symbol, Optional<Symbol>> outputToInputTranslator(UnionNode node, int sourceIndex) { return symbol -> Optional.of(node.getSymbolMapping().get(symbol).get(sourceIndex)); }
@Override public PlanNodeCostEstimate visitUnion(UnionNode node, Void context) { // this assumes that all union inputs will be gathered over the network // that is not aways true // but this estimate is better that returning UNKNOWN, as it sets // cumulative cost to unknown double inputSizeInBytes = getStats(node).getOutputSizeInBytes(node.getOutputSymbols(), types); return calculateRemoteGatherCost(inputSizeInBytes); }
/** * Returns the input to output symbol mapping for the given source channel. * A single input symbol can map to multiple output symbols, thus requiring a Multimap. */ public Multimap<Symbol, QualifiedNameReference> outputSymbolMap(int sourceIndex) { return Multimaps.transformValues(FluentIterable.from(getOutputSymbols()) .toMap(outputToSourceSymbolFunction(sourceIndex)) .asMultimap() .inverse(), Symbol::toQualifiedNameReference); }
public UnionNode union(ListMultimap<Symbol, Symbol> outputsToInputs, List<PlanNode> sources) { ImmutableList<Symbol> outputs = outputsToInputs.keySet().stream().collect(toImmutableList()); return new UnionNode(idAllocator.getNextId(), sources, outputsToInputs, outputs); }
private PlanNode pushProjectionThrough(ProjectNode node, UnionNode source) { // OutputLayout of the resultant Union, will be same as the layout of the Project List<Symbol> outputLayout = node.getOutputSymbols(); // Mapping from the output symbol to ordered list of symbols from each of the sources ImmutableListMultimap.Builder<Symbol, Symbol> mappings = ImmutableListMultimap.builder(); // sources for the resultant UnionNode ImmutableList.Builder<PlanNode> outputSources = ImmutableList.builder(); for (int i = 0; i < source.getSources().size(); i++) { Map<Symbol, QualifiedNameReference> outputToInput = source.sourceSymbolMap(i); // Map: output of union -> input of this source to the union ImmutableMap.Builder<Symbol, Expression> assignments = ImmutableMap.builder(); // assignments for the new ProjectNode // mapping from current ProjectNode to new ProjectNode, used to identify the output layout Map<Symbol, Symbol> projectSymbolMapping = new HashMap<>(); // Translate the assignments in the ProjectNode using symbols of the source of the UnionNode for (Map.Entry<Symbol, Expression> entry : node.getAssignments().entrySet()) { Expression translatedExpression = translateExpression(entry.getValue(), outputToInput); Type type = symbolAllocator.getTypes().get(entry.getKey()); Symbol symbol = symbolAllocator.newSymbol(translatedExpression, type); assignments.put(symbol, translatedExpression); projectSymbolMapping.put(entry.getKey(), symbol); } outputSources.add(new ProjectNode(idAllocator.getNextId(), source.getSources().get(i), assignments.build())); outputLayout.forEach(symbol -> mappings.put(symbol, projectSymbolMapping.get(symbol))); } return new UnionNode(node.getId(), outputSources.build(), mappings.build(), ImmutableList.copyOf(mappings.build().keySet())); }