@Override public Integer visitExchange(ExchangeNode node, Void context) { if (node.getScope() == ExchangeNode.Scope.REMOTE) { return 0; } return visitPlan(node, context); }
@Override public Integer visitExchange(ExchangeNode node, Void context) { if (node.getScope() == ExchangeNode.Scope.REMOTE) { return 0; } return visitPlan(node, context); }
@Override public Set<PlanFragmentId> visitExchange(ExchangeNode node, PlanFragmentId currentFragmentId) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the phased execution scheduler"); ImmutableSet.Builder<PlanFragmentId> allSources = ImmutableSet.builder(); // Link the source fragments together, so we only schedule one at a time. Set<PlanFragmentId> previousSources = ImmutableSet.of(); for (PlanNode subPlanNode : node.getSources()) { Set<PlanFragmentId> currentSources = subPlanNode.accept(this, currentFragmentId); allSources.addAll(currentSources); addEdges(previousSources, currentSources); previousSources = currentSources; } return allSources.build(); }
@Override public Set<PlanFragmentId> visitExchange(ExchangeNode node, PlanFragmentId currentFragmentId) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the phased execution scheduler"); ImmutableSet.Builder<PlanFragmentId> allSources = ImmutableSet.builder(); // Link the source fragments together, so we only schedule one at a time. Set<PlanFragmentId> previousSources = ImmutableSet.of(); for (PlanNode subPlanNode : node.getSources()) { Set<PlanFragmentId> currentSources = subPlanNode.accept(this, currentFragmentId); allSources.addAll(currentSources); addEdges(previousSources, currentSources); previousSources = currentSources; } return allSources.build(); }
@Override public PhysicalOperation visitExchange(ExchangeNode node, LocalExecutionPlanContext context) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the local planner"); if (node.getOrderingScheme().isPresent()) { return createLocalMerge(node, context); } return createLocalExchange(node, context); }
@Override public PhysicalOperation visitExchange(ExchangeNode node, LocalExecutionPlanContext context) { checkArgument(node.getScope() == LOCAL, "Only local exchanges are supported in the local planner"); if (node.getOrderingScheme().isPresent()) { return createLocalMerge(node, context); } return createLocalExchange(node, context); }
@Override public boolean shapeMatches(PlanNode node) { if (!(node instanceof ExchangeNode)) { return false; } ExchangeNode exchangeNode = (ExchangeNode) node; return exchangeNode.getScope() == scope && exchangeNode.getType() == type; }
private static boolean isRemoteExchange(PlanNode planNode) { return (planNode instanceof ExchangeNode) && ((ExchangeNode) planNode).getScope().equals(REMOTE); } }
private static boolean isRemoteExchange(PlanNode planNode) { return (planNode instanceof ExchangeNode) && ((ExchangeNode) planNode).getScope().equals(REMOTE); } }
@Override public boolean shapeMatches(PlanNode node) { if (!(node instanceof ExchangeNode)) { return false; } ExchangeNode exchangeNode = (ExchangeNode) node; return exchangeNode.getScope() == scope && exchangeNode.getType() == type; }
@Override public Optional<SeenExchanges> visitExchange(ExchangeNode node, Void context) { Optional<SeenExchanges> seenExchangesOptional = aggregatedSeenExchanges(node.getSources()); if (!seenExchangesOptional.isPresent()) { // No partial aggregation below return Optional.empty(); } if (!node.getType().equals(REPARTITION)) { return seenExchangesOptional; } SeenExchanges seenExchanges = seenExchangesOptional.get(); if (node.getScope().equals(REMOTE)) { return Optional.of(new SeenExchanges(false, true)); } return Optional.of(new SeenExchanges(true, seenExchanges.remoteRepartitionExchange)); }
@Override public Optional<SeenExchanges> visitExchange(ExchangeNode node, Void context) { Optional<SeenExchanges> seenExchangesOptional = aggregatedSeenExchanges(node.getSources()); if (!seenExchangesOptional.isPresent()) { // No partial aggregation below return Optional.empty(); } if (!node.getType().equals(REPARTITION)) { return seenExchangesOptional; } SeenExchanges seenExchanges = seenExchangesOptional.get(); if (node.getScope().equals(REMOTE)) { return Optional.of(new SeenExchanges(false, true)); } return Optional.of(new SeenExchanges(true, seenExchanges.remoteRepartitionExchange)); }
@Override public PlanWithProperties visitExchange(ExchangeNode node, StreamPreferredProperties parentPreferences) { checkArgument(node.getScope() != LOCAL, "AddLocalExchanges can not process a plan containing a local exchange"); // this node changes the input organization completely, so we do not pass through parent preferences if (node.getOrderingScheme().isPresent()) { return planAndEnforceChildren( node, any().withOrderSensitivity(), any().withOrderSensitivity()); } return planAndEnforceChildren(node, any(), defaultParallelism(session)); }
@Override public PlanWithProperties visitExchange(ExchangeNode node, StreamPreferredProperties parentPreferences) { checkArgument(node.getScope() != LOCAL, "AddLocalExchanges can not process a plan containing a local exchange"); // this node changes the input organization completely, so we do not pass through parent preferences if (node.getOrderingScheme().isPresent()) { return planAndEnforceChildren( node, any().withOrderSensitivity(), any().withOrderSensitivity()); } return planAndEnforceChildren(node, any(), defaultParallelism(session)); }
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }
@Override public Void visitExchange(ExchangeNode node, Integer indent) { Partitioning partitioning = node.getPartitioningScheme().getPartitioning(); output( indent, "%s exchange (%s, %s, %s)", node.getScope().name().toLowerCase(ENGLISH), node.getType(), partitioning.getHandle(), partitioning.getArguments().stream() .map(Object::toString) .sorted() // Currently, order of hash columns is not deterministic .collect(joining(", ", "[", "]"))); return visitPlan(node, indent + 1); }
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
private Consumer<Plan> assertRemoteExchangesCount(int expectedRemoteExchangesCount) { return plan -> { int actualRemoteExchangesCount = searchFrom(plan.getRoot()) .where(node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope() == ExchangeNode.Scope.REMOTE) .findAll() .size(); if (actualRemoteExchangesCount != expectedRemoteExchangesCount) { Session session = getSession(); Metadata metadata = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getMetadata(); String formattedPlan = textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); throw new AssertionError(format( "Expected [\n%s\n] remote exchanges but found [\n%s\n] remote exchanges. Actual plan is [\n\n%s\n]", expectedRemoteExchangesCount, actualRemoteExchangesCount, formattedPlan)); } }; }
private Consumer<Plan> assertRemoteExchangesCount(int expectedRemoteExchangesCount) { return plan -> { int actualRemoteExchangesCount = searchFrom(plan.getRoot()) .where(node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope() == ExchangeNode.Scope.REMOTE) .findAll() .size(); if (actualRemoteExchangesCount != expectedRemoteExchangesCount) { Session session = getSession(); Metadata metadata = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getMetadata(); String formattedPlan = textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); throw new AssertionError(format( "Expected [\n%s\n] remote exchanges but found [\n%s\n] remote exchanges. Actual plan is [\n\n%s\n]", expectedRemoteExchangesCount, actualRemoteExchangesCount, formattedPlan)); } }; }