@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(Symbol::toSymbolReference) .map(ArgumentBinding::expressionBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(ArgumentBinding::columnBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
@Override public boolean shapeMatches(PlanNode node) { if (!(node instanceof ExchangeNode)) { return false; } ExchangeNode exchangeNode = (ExchangeNode) node; return exchangeNode.getScope() == scope && exchangeNode.getType() == type; }
@Override public boolean shapeMatches(PlanNode node) { if (!(node instanceof ExchangeNode)) { return false; } ExchangeNode exchangeNode = (ExchangeNode) node; return exchangeNode.getScope() == scope && exchangeNode.getType() == type; }
@Override public Optional<SeenExchanges> visitExchange(ExchangeNode node, Void context) { Optional<SeenExchanges> seenExchangesOptional = aggregatedSeenExchanges(node.getSources()); if (!seenExchangesOptional.isPresent()) { // No partial aggregation below return Optional.empty(); } if (!node.getType().equals(REPARTITION)) { return seenExchangesOptional; } SeenExchanges seenExchanges = seenExchangesOptional.get(); if (node.getScope().equals(REMOTE)) { return Optional.of(new SeenExchanges(false, true)); } return Optional.of(new SeenExchanges(true, seenExchanges.remoteRepartitionExchange)); }
@Override public Optional<SeenExchanges> visitExchange(ExchangeNode node, Void context) { Optional<SeenExchanges> seenExchangesOptional = aggregatedSeenExchanges(node.getSources()); if (!seenExchangesOptional.isPresent()) { // No partial aggregation below return Optional.empty(); } if (!node.getType().equals(REPARTITION)) { return seenExchangesOptional; } SeenExchanges seenExchanges = seenExchangesOptional.get(); if (node.getScope().equals(REMOTE)) { return Optional.of(new SeenExchanges(false, true)); } return Optional.of(new SeenExchanges(true, seenExchanges.remoteRepartitionExchange)); }
@Test public void testUnionOverSingleNodeAggregationAndUnion() { Plan plan = plan( "SELECT count(*) FROM (" + "SELECT 1 FROM nation GROUP BY regionkey " + "UNION ALL (" + " SELECT 1 FROM nation " + " UNION ALL " + " SELECT 1 FROM nation))", LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); List<PlanNode> remotes = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll(); assertEquals(remotes.size(), 2, "There should be exactly two RemoteExchanges"); assertEquals(((ExchangeNode) remotes.get(0)).getType(), GATHER); assertEquals(((ExchangeNode) remotes.get(1)).getType(), REPARTITION); }
private static boolean isRemoteGatheringExchange(PlanNode planNode) { return isRemoteExchange(planNode) && ((ExchangeNode) planNode).getType().equals(GATHER); }
@Test public void testUnionOverSingleNodeAggregationAndUnion() { Plan plan = plan( "SELECT count(*) FROM (" + "SELECT 1 FROM nation GROUP BY regionkey " + "UNION ALL (" + " SELECT 1 FROM nation " + " UNION ALL " + " SELECT 1 FROM nation))", LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); List<PlanNode> remotes = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll(); assertEquals(remotes.size(), 2, "There should be exactly two RemoteExchanges"); assertEquals(((ExchangeNode) remotes.get(0)).getType(), GATHER); assertEquals(((ExchangeNode) remotes.get(1)).getType(), REPARTITION); }
private static boolean isRemoteGatheringExchange(PlanNode planNode) { return isRemoteExchange(planNode) && ((ExchangeNode) planNode).getType().equals(GATHER); }
@Test public void testSimpleUnion() { Plan plan = plan( "SELECT suppkey FROM supplier UNION ALL SELECT nationkey FROM nation", LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); List<PlanNode> remotes = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll(); assertEquals(remotes.size(), 1, "There should be exactly one RemoteExchange"); assertEquals(((ExchangeNode) Iterables.getOnlyElement(remotes)).getType(), GATHER); assertPlanIsFullyDistributed(plan); }
@Test public void testSimpleUnion() { Plan plan = plan( "SELECT suppkey FROM supplier UNION ALL SELECT nationkey FROM nation", LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); List<PlanNode> remotes = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll(); assertEquals(remotes.size(), 1, "There should be exactly one RemoteExchange"); assertEquals(((ExchangeNode) Iterables.getOnlyElement(remotes)).getType(), GATHER); assertPlanIsFullyDistributed(plan); }
@Override public Void visitExchange(ExchangeNode node, Integer indent) { Partitioning partitioning = node.getPartitioningScheme().getPartitioning(); output( indent, "%s exchange (%s, %s, %s)", node.getScope().name().toLowerCase(ENGLISH), node.getType(), partitioning.getHandle(), partitioning.getArguments().stream() .map(Object::toString) .sorted() // Currently, order of hash columns is not deterministic .collect(joining(", ", "[", "]"))); return visitPlan(node, indent + 1); }
@Test public void testUnionUnderTopN() { Plan plan = plan( "SELECT * FROM (" + " SELECT regionkey FROM nation " + " UNION ALL " + " SELECT nationkey FROM nation" + ") t(a) " + "ORDER BY a LIMIT 1", LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); List<PlanNode> remotes = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll(); assertEquals(remotes.size(), 1, "There should be exactly one RemoteExchange"); assertEquals(((ExchangeNode) Iterables.getOnlyElement(remotes)).getType(), GATHER); int numberOfpartialTopN = searchFrom(plan.getRoot()) .where(planNode -> planNode instanceof TopNNode && ((TopNNode) planNode).getStep().equals(TopNNode.Step.PARTIAL)) .count(); assertEquals(numberOfpartialTopN, 2, "There should be exactly two partial TopN nodes"); assertPlanIsFullyDistributed(plan); }
@Override public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType() == ExchangeNode.Type.GATHER) { return rebaseAndDeriveProperties(node, child); } // Always add an exchange because ExplainAnalyze should be in its own stage child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType() == ExchangeNode.Type.GATHER) { return rebaseAndDeriveProperties(node, child); } // Always add an exchange because ExplainAnalyze should be in its own stage child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitStatisticsWriterNode(StatisticsWriterNode node, PreferredProperties context) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { return rebaseAndDeriveProperties(node, child); } if (!child.getProperties().isCoordinatorOnly()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitTableFinish(TableFinishNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { return rebaseAndDeriveProperties(node, child); } if (!child.getProperties().isCoordinatorOnly()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitTableFinish(TableFinishNode node, PreferredProperties preferredProperties) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { return rebaseAndDeriveProperties(node, child); } if (!child.getProperties().isCoordinatorOnly()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitStatisticsWriterNode(StatisticsWriterNode node, PreferredProperties context) { PlanWithProperties child = planChild(node, PreferredProperties.any()); // if the child is already a gathering exchange, don't add another if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { return rebaseAndDeriveProperties(node, child); } if (!child.getProperties().isCoordinatorOnly()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }