for (int i = 0; i < node.getSources().size(); i++) { Map<Symbol, SymbolReference> outputsToInputs = new HashMap<>(); for (int index = 0; index < node.getInputs().get(i).size(); index++) { outputsToInputs.put( node.getOutputSymbols().get(index), node.getInputs().get(i).get(index).toSymbolReference()); PlanNode source = node.getSources().get(i); PlanNode rewrittenSource = context.rewrite(source, sourcePredicate); if (rewrittenSource != source) { return new ExchangeNode( node.getId(), node.getType(), node.getScope(), node.getPartitioningScheme(), builder.build(), node.getInputs(), node.getOrderingScheme());
public static ExchangeNode partitionedExchange(PlanNodeId id, Scope scope, PlanNode child, List<Symbol> partitioningColumns, Optional<Symbol> hashColumns) { return partitionedExchange(id, scope, child, partitioningColumns, hashColumns, false); }
public static ExchangeNode partitionedExchange(PlanNodeId id, Scope scope, PlanNode child, PartitioningScheme partitioningScheme) { if (partitioningScheme.getPartitioning().getHandle().isSingleNode()) { return gatheringExchange(id, scope, child); } return new ExchangeNode( id, ExchangeNode.Type.REPARTITION, scope, partitioningScheme, ImmutableList.of(child), ImmutableList.of(partitioningScheme.getOutputLayout()).asList(), Optional.empty()); }
@Override public PlanNode replaceChildren(List<PlanNode> newChildren) { return new ExchangeNode(getId(), type, scope, partitioningScheme, newChildren, inputs, orderingScheme); } }
public static Map<Symbol, Symbol> exchangeInputToOutput(ExchangeNode node, int sourceIndex) { List<Symbol> inputSymbols = node.getInputs().get(sourceIndex); Map<Symbol, Symbol> inputToOutput = new HashMap<>(); for (int i = 0; i < node.getOutputSymbols().size(); i++) { inputToOutput.put(inputSymbols.get(i), node.getOutputSymbols().get(i)); } return inputToOutput; }
@Override public Void visitExchange(ExchangeNode node, Set<Symbol> boundSymbols) { for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); checkDependencies(subplan.getOutputSymbols(), node.getInputs().get(i), "EXCHANGE subplan must provide all of the necessary symbols"); subplan.accept(this, boundSymbols); // visit child } checkDependencies(node.getOutputSymbols(), node.getPartitioningScheme().getOutputLayout(), "EXCHANGE must provide all of the necessary symbols for partition function"); return null; }
@Override public PlanNode visitExchange(ExchangeNode node, RewriteContext<Expression> context) { boolean modified = false; ImmutableList.Builder<PlanNode> builder = ImmutableList.builder(); for (int i = 0; i < node.getSources().size(); i++) { Map<Symbol, QualifiedNameReference> outputsToInputs = new HashMap<>(); for (int index = 0; index < node.getInputs().get(i).size(); index++) { outputsToInputs.put( node.getOutputSymbols().get(index), node.getInputs().get(i).get(index).toQualifiedNameReference()); } Expression sourcePredicate = ExpressionTreeRewriter.rewriteWith(new ExpressionSymbolInliner(outputsToInputs), context.get()); PlanNode source = node.getSources().get(i); PlanNode rewrittenSource = context.rewrite(source, sourcePredicate); if (rewrittenSource != source) { modified = true; } builder.add(rewrittenSource); } if (modified) { return new ExchangeNode( node.getId(), node.getType(), node.getPartitionFunction(), builder.build(), node.getInputs()); } return node; }
private PlanNode pushPartial(AggregationNode aggregation, ExchangeNode exchange, Context context) for (int i = 0; i < exchange.getSources().size(); i++) { PlanNode source = exchange.getSources().get(i); for (int outputIndex = 0; outputIndex < exchange.getOutputSymbols().size(); outputIndex++) { Symbol output = exchange.getOutputSymbols().get(outputIndex); Symbol input = exchange.getInputs().get(i).get(outputIndex); if (!output.equals(input)) { mappingsBuilder.put(output, input); exchange.getPartitioningScheme().getPartitioning(), aggregation.getOutputSymbols(), exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition()); return new ExchangeNode( context.getIdAllocator().getNextId(), exchange.getType(), exchange.getScope(), partitioning, partials,
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
Set<Symbol> partitioningColumns = exchange.getPartitioningScheme().getPartitioning().getColumns(); for (int i = 0; i < exchange.getSources().size(); i++) { Map<Symbol, SymbolReference> outputToInputMap = extractExchangeOutputToInput(exchange, i); if (exchange.getPartitioningScheme().getHashColumn().isPresent()) { projections.put(exchange.getPartitioningScheme().getHashColumn().get(), exchange.getPartitioningScheme().getHashColumn().get().toSymbolReference()); inputs.add(exchange.getPartitioningScheme().getHashColumn().get()); if (exchange.getOrderingScheme().isPresent()) { exchange.getOrderingScheme().get().getOrderBy().stream() newSourceBuilder.add(new ProjectNode(context.getIdAllocator().getNextId(), exchange.getSources().get(i), projections.build())); inputsBuilder.add(inputs.build()); exchange.getPartitioningScheme().getHashColumn().ifPresent(outputBuilder::add); if (exchange.getOrderingScheme().isPresent()) { exchange.getOrderingScheme().get().getOrderBy().stream() .filter(symbol -> !partitioningColumns.contains(symbol)) .forEach(outputBuilder::add); exchange.getPartitioningScheme().getPartitioning(), outputBuilder.build(), exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
@Override public StreamProperties visitExchange(ExchangeNode node, List<StreamProperties> inputProperties) { if (node.getOrderingScheme().isPresent()) { return StreamProperties.ordered(); } if (node.getScope() == REMOTE) { // TODO: correctly determine if stream is parallelised // based on session properties return StreamProperties.fixedStreams(); } switch (node.getType()) { case GATHER: return StreamProperties.singleStream(); case REPARTITION: if (node.getPartitioningScheme().getPartitioning().getHandle().equals(FIXED_ARBITRARY_DISTRIBUTION)) { return new StreamProperties(FIXED, Optional.empty(), false); } return new StreamProperties( FIXED, Optional.of(node.getPartitioningScheme().getPartitioning().getArguments().stream() .map(ArgumentBinding::getColumn) .collect(toImmutableList())), false); case REPLICATE: return new StreamProperties(MULTIPLE, Optional.empty(), false); } throw new UnsupportedOperationException("not yet implemented"); }
@Override public Void visitExchange(ExchangeNode node, Integer indent) if (node.getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getOrderingScheme().get(); List<String> orderBy = orderingScheme.getOrderBy() .stream() UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, node.getScope().toString()), Joiner.on(", ").join(orderBy), formatOutputs(node.getOutputSymbols())); else if (node.getScope() == Scope.LOCAL) { print(indent, "- LocalExchange[%s%s]%s (%s) => %s", node.getPartitioningScheme().getPartitioning().getHandle(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), Joiner.on(", ").join(node.getPartitioningScheme().getPartitioning().getArguments()), formatOutputs(node.getOutputSymbols())); UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, node.getScope().toString()), node.getType(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), formatOutputs(node.getOutputSymbols())); printStats(indent + 2, node.getId());
if (node.getType() == ExchangeNode.Type.GATHER) { driverInstanceCount = 1; context.setDriverInstanceCount(1); List<Integer> channels = node.getPartitioningScheme().getPartitioning().getArguments().stream() .map(argument -> node.getOutputSymbols().indexOf(argument.getColumn())) .collect(toImmutableList()); Optional<Integer> hashChannel = node.getPartitioningScheme().getHashColumn() .map(symbol -> node.getOutputSymbols().indexOf(symbol)); for (int i = 0; i < node.getSources().size(); i++) { PlanNode sourceNode = node.getSources().get(i); node.getPartitioningScheme().getPartitioning().getHandle(), driverInstanceCount, types, exchangeSourcePipelineExecutionStrategy, maxLocalExchangeBufferSize); for (int i = 0; i < node.getSources().size(); i++) { DriverFactoryParameters driverFactoryParameters = driverFactoryParametersList.get(i); PhysicalOperation source = driverFactoryParameters.getSource(); LocalExecutionPlanContext subContext = driverFactoryParameters.getSubContext(); List<Symbol> expectedLayout = node.getInputs().get(i); Function<Page, Page> pagePreprocessor = enforceLayoutProcessor(expectedLayout, source.getLayout()); List<OperatorFactory> operatorFactories = new ArrayList<>(source.getOperatorFactories()); localExchangeFactory,
if (!right.getProperties().isSingleNode()) { right = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); replicatedExchange(idAllocator.getNextId(), REMOTE, right.getNode()), right.getProperties()); partitionedExchange(idAllocator.getNextId(), REMOTE, left.getNode(), ImmutableList.of(node.getLeftPartitionSymbol().get()), Optional.empty()), left.getProperties()); right = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, right.getNode(), ImmutableList.of(node.getRightPartitionSymbol().get()), Optional.empty()), right.getProperties());
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(Symbol::toSymbolReference) .map(ArgumentBinding::expressionBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
@Override public PlanNode visitExchange(ExchangeNode node, List<PlanNode> newChildren) { return new ExchangeNode( node.getId(), node.getType(), node.getPartitionFunction(), newChildren, node.getInputs()); }
@Override public PlanNodeCostEstimate visitExchange(ExchangeNode node, Void context) double inputSizeInBytes = getStats(node).getOutputSizeInBytes(node.getOutputSymbols(), types); switch (node.getScope()) { case LOCAL: switch (node.getType()) { case GATHER: return PlanNodeCostEstimate.zero(); return PlanNodeCostEstimate.zero(); default: throw new IllegalArgumentException("Unexpected type: " + node.getType()); switch (node.getType()) { case GATHER: return calculateRemoteGatherCost(inputSizeInBytes); throw new IllegalArgumentException("Unexpected type: " + node.getType()); throw new IllegalArgumentException("Unexpected scope: " + node.getScope());
ExchangeNode exchangeNode = gatheringExchange(idAllocator.getNextId(), LOCAL, planWithProperties.getNode()); return deriveProperties(exchangeNode, planWithProperties.getProperties()); if (!requiredPartitionColumns.isPresent()) { ExchangeNode exchangeNode = partitionedExchange( idAllocator.getNextId(), LOCAL, ExchangeNode exchangeNode = partitionedExchange( idAllocator.getNextId(), LOCAL, ExchangeNode exchangeNode = gatheringExchange( idAllocator.getNextId(), LOCAL,
checkArgument(node.getOrderingScheme().isPresent(), "orderingScheme is absent"); checkState(node.getSources().size() == 1, "single source is expected"); PlanNode sourceNode = getOnlyElement(node.getSources()); LocalExecutionPlanContext subContext = context.createSubContext(); PhysicalOperation source = sourceNode.accept(this, subContext); List<Type> types = getSourceOperatorTypes(node, context.getTypes()); LocalExchangeFactory exchangeFactory = new LocalExchangeFactory( node.getPartitioningScheme().getPartitioning().getHandle(), operatorsCount, types, List<Symbol> expectedLayout = node.getInputs().get(0); Function<Page, Page> pagePreprocessor = enforceLayoutProcessor(expectedLayout, source.getLayout()); operatorFactories.add(new LocalExchangeSinkOperatorFactory( exchangeFactory, subContext.getNextOperatorId(), node.getId(), exchangeFactory.newSinkFactoryId(), pagePreprocessor)); OrderingScheme orderingScheme = node.getOrderingScheme().get(); ImmutableMap<Symbol, Integer> layout = makeLayout(node); List<Integer> sortChannels = getChannelsForSymbols(orderingScheme.getOrderBy(), layout); OperatorFactory operatorFactory = new LocalMergeSourceOperatorFactory( context.getNextOperatorId(), node.getId(),
for (int i = 0; i < exchange.getSources().size(); i++) { Map<Symbol, QualifiedNameReference> outputToInputMap = extractExchangeOutputToInput(exchange, i); exchange.getPartitionFunction().getPartitionFunctionArguments().stream() .filter(PartitionFunctionArgumentBinding::isVariable) .map(PartitionFunctionArgumentBinding::getColumn) if (exchange.getPartitionFunction().getHashColumn().isPresent()) { projections.put(exchange.getPartitionFunction().getHashColumn().get(), exchange.getPartitionFunction().getHashColumn().get().toQualifiedNameReference()); inputs.add(exchange.getPartitionFunction().getHashColumn().get()); newSourceBuilder.add(new ProjectNode(idAllocator.getNextId(), exchange.getSources().get(i), projections)); inputsBuilder.add(inputs.build()); exchange.getPartitionFunction().getPartitionFunctionArguments().stream() .filter(PartitionFunctionArgumentBinding::isVariable) .map(PartitionFunctionArgumentBinding::getColumn) .forEach(outputBuilder::add); if (exchange.getPartitionFunction().getHashColumn().isPresent()) { outputBuilder.add(exchange.getPartitionFunction().getHashColumn().get()); exchange.getPartitionFunction().getPartitioningHandle(), outputBuilder.build(), exchange.getPartitionFunction().getPartitionFunctionArguments(), exchange.getPartitionFunction().getHashColumn(), exchange.getPartitionFunction().isReplicateNulls(), exchange.getPartitionFunction().getBucketToPartition());