@Override public PhysicalOperation visitExchange(ExchangeNode node, LocalExecutionPlanContext context) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the local planner"); if (node.getOrderingScheme().isPresent()) { return createLocalMerge(node, context); } return createLocalExchange(node, context); }
@Override public PhysicalOperation visitExchange(ExchangeNode node, LocalExecutionPlanContext context) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the local planner"); if (node.getOrderingScheme().isPresent()) { return createLocalMerge(node, context); } return createLocalExchange(node, context); }
@Override public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) { checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName()); ExchangeNode exchangeNode = (ExchangeNode) node; if (!orderBy.isEmpty()) { if (!exchangeNode.getOrderingScheme().isPresent()) { return NO_MATCH; } if (!orderingSchemeMatches(orderBy, exchangeNode.getOrderingScheme().get(), symbolAliases)) { return NO_MATCH; } } return MatchResult.match(); }
@Override public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) { checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName()); ExchangeNode exchangeNode = (ExchangeNode) node; if (!orderBy.isEmpty()) { if (!exchangeNode.getOrderingScheme().isPresent()) { return NO_MATCH; } if (!orderingSchemeMatches(orderBy, exchangeNode.getOrderingScheme().get(), symbolAliases)) { return NO_MATCH; } } return MatchResult.match(); }
node.getPartitioningScheme().getPartitioning().getColumns().stream() .forEach(expectedOutputSymbols::add); node.getOrderingScheme().ifPresent(orderingScheme -> expectedOutputSymbols.addAll(orderingScheme.getOrderBy())); rewrittenSources.build(), inputsBySource, node.getOrderingScheme());
node.getPartitioningScheme().getPartitioning().getColumns().stream() .forEach(expectedOutputSymbols::add); node.getOrderingScheme().ifPresent(orderingScheme -> expectedOutputSymbols.addAll(orderingScheme.getOrderBy())); rewrittenSources.build(), inputsBySource, node.getOrderingScheme());
@Override public PlanWithProperties visitExchange(ExchangeNode node, StreamPreferredProperties parentPreferences) { checkArgument(node.getScope() != LOCAL, "AddLocalExchanges can not process a plan containing a local exchange"); // this node changes the input organization completely, so we do not pass through parent preferences if (node.getOrderingScheme().isPresent()) { return planAndEnforceChildren( node, any().withOrderSensitivity(), any().withOrderSensitivity()); } return planAndEnforceChildren(node, any(), defaultParallelism(session)); }
builder.build(), node.getInputs(), node.getOrderingScheme());
builder.build(), node.getInputs(), node.getOrderingScheme());
node.getPartitioningScheme().getBucketToPartition()); Optional<OrderingScheme> orderingScheme = node.getOrderingScheme().map(this::canonicalizeAndDistinct);
@Override public PlanWithProperties visitExchange(ExchangeNode node, StreamPreferredProperties parentPreferences) { checkArgument(node.getScope() != LOCAL, "AddLocalExchanges can not process a plan containing a local exchange"); // this node changes the input organization completely, so we do not pass through parent preferences if (node.getOrderingScheme().isPresent()) { return planAndEnforceChildren( node, any().withOrderSensitivity(), any().withOrderSensitivity()); } return planAndEnforceChildren(node, any(), defaultParallelism(session)); }
@Override public StreamProperties visitExchange(ExchangeNode node, List<StreamProperties> inputProperties) { if (node.getOrderingScheme().isPresent()) { return StreamProperties.ordered(); } if (node.getScope() == REMOTE) { // TODO: correctly determine if stream is parallelised // based on session properties return StreamProperties.fixedStreams(); } switch (node.getType()) { case GATHER: return StreamProperties.singleStream(); case REPARTITION: if (node.getPartitioningScheme().getPartitioning().getHandle().equals(FIXED_ARBITRARY_DISTRIBUTION)) { return new StreamProperties(FIXED, Optional.empty(), false); } return new StreamProperties( FIXED, Optional.of(node.getPartitioningScheme().getPartitioning().getArguments().stream() .map(ArgumentBinding::getColumn) .collect(toImmutableList())), false); case REPLICATE: return new StreamProperties(MULTIPLE, Optional.empty(), false); } throw new UnsupportedOperationException("not yet implemented"); }
@Override public StreamProperties visitExchange(ExchangeNode node, List<StreamProperties> inputProperties) { if (node.getOrderingScheme().isPresent()) { return StreamProperties.ordered(); } if (node.getScope() == REMOTE) { // TODO: correctly determine if stream is parallelised // based on session properties return StreamProperties.fixedStreams(); } switch (node.getType()) { case GATHER: return StreamProperties.singleStream(); case REPARTITION: if (node.getPartitioningScheme().getPartitioning().getHandle().equals(FIXED_ARBITRARY_DISTRIBUTION)) { return new StreamProperties(FIXED, Optional.empty(), false); } return new StreamProperties( FIXED, Optional.of(node.getPartitioningScheme().getPartitioning().getArguments().stream() .map(ArgumentBinding::getColumn) .collect(toImmutableList())), false); case REPLICATE: return new StreamProperties(MULTIPLE, Optional.empty(), false); } throw new UnsupportedOperationException("not yet implemented"); }
node.getPartitioningScheme().getBucketToPartition()); Optional<OrderingScheme> orderingScheme = node.getOrderingScheme().map(this::canonicalizeAndDistinct);
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }
@Override public Void visitExchange(ExchangeNode node, Integer indent) if (node.getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getOrderingScheme().get(); List<String> orderBy = orderingScheme.getOrderBy() .stream()
@Override public Void visitExchange(ExchangeNode node, Integer indent) if (node.getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getOrderingScheme().get(); List<String> orderBy = orderingScheme.getOrderBy() .stream()
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }