public String getJsonPlan(Session session, Statement statement, Type planType, List<Expression> parameters, WarningCollector warningCollector) { DataDefinitionTask<?> task = dataDefinitionTask.get(statement.getClass()); if (task != null) { // todo format as json return explainTask(statement, task, parameters); } switch (planType) { case IO: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); return textIOPlan(plan.getRoot(), metadata, session); default: throw new PrestoException(NOT_SUPPORTED, format("Unsupported explain plan type %s for JSON format", planType)); } }
private static List<OptionalDouble> getEstimatedValuesInternal(List<Metric> metrics, String query, QueryRunner runner, Session session) // TODO inline back this method { Plan queryPlan = runner.createPlan(session, query, WarningCollector.NOOP); OutputNode outputNode = (OutputNode) queryPlan.getRoot(); PlanNodeStatsEstimate outputNodeStats = queryPlan.getStatsAndCosts().getStats().getOrDefault(queryPlan.getRoot().getId(), PlanNodeStatsEstimate.unknown()); StatsContext statsContext = buildStatsContext(queryPlan, outputNode); return getEstimatedValues(metrics, outputNodeStats, statsContext); }
private Constraint<ColumnHandle> getConstraint(Plan plan) { Optional<TableScanNode> scanNode = searchFrom(plan.getRoot()) .where(TableScanNode.class::isInstance) .findSingle(); if (!scanNode.isPresent()) { return Constraint.alwaysFalse(); } return new Constraint<>(scanNode.get().getCurrentConstraint()); }
public String getGraphvizPlan(Session session, Statement statement, Type planType, List<Expression> parameters, WarningCollector warningCollector) { DataDefinitionTask<?> task = dataDefinitionTask.get(statement.getClass()); if (task != null) { // todo format as graphviz return explainTask(statement, task, parameters); } switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); return PlanPrinter.graphvizLogicalPlan(plan.getRoot(), plan.getTypes()); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); return PlanPrinter.graphvizDistributedPlan(subPlan); } throw new IllegalArgumentException("Unhandled plan type: " + planType); }
public String getPlan(Session session, Statement statement, Type planType, List<Expression> parameters, WarningCollector warningCollector) { DataDefinitionTask<?> task = dataDefinitionTask.get(statement.getClass()); if (task != null) { return explainTask(statement, task, parameters); } switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); return PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionRegistry(), plan.getStatsAndCosts(), session, 0, false); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); return PlanPrinter.textDistributedPlan(subPlan, metadata.getFunctionRegistry(), session, false); case IO: return IOPlanPrinter.textIOPlan(getLogicalPlan(session, statement, parameters, warningCollector).getRoot(), metadata, session); } throw new IllegalArgumentException("Unhandled plan type: " + planType); }
private String generateQueryPlan(String query) { String sql = query.replaceAll("\\s+;\\s+$", "") .replace("${database}.${schema}.", "") .replace("\"${database}\".\"${schema}\".\"${prefix}", "\""); Plan plan = plan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); JoinOrderPrinter joinOrderPrinter = new JoinOrderPrinter(); plan.getRoot().accept(joinOrderPrinter, 0); return joinOrderPrinter.result(); }
private void assertPlanIsFullyDistributed(Plan plan) { int numberOfGathers = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteGatheringExchange) .findAll() .size(); if (numberOfGathers == 0) { // there are no "gather" nodes, so the plan is expected to be fully distributed return; } assertTrue( searchFrom(plan.getRoot()) .recurseOnlyWhen(TestUnion::isNotRemoteGatheringExchange) .findAll() .stream() .noneMatch(this::shouldBeDistributed), "There is a node that should be distributed between output and first REMOTE GATHER ExchangeNode"); assertEquals(numberOfGathers, 1, "Only a single REMOTE GATHER was expected"); }
@Test public void testSimpleUnion() { Plan plan = plan( "SELECT suppkey FROM supplier UNION ALL SELECT nationkey FROM nation", LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); List<PlanNode> remotes = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll(); assertEquals(remotes.size(), 1, "There should be exactly one RemoteExchange"); assertEquals(((ExchangeNode) Iterables.getOnlyElement(remotes)).getType(), GATHER); assertPlanIsFullyDistributed(plan); }
@Test public void testUnionOverSingleNodeAggregationAndUnion() { Plan plan = plan( "SELECT count(*) FROM (" + "SELECT 1 FROM nation GROUP BY regionkey " + "UNION ALL (" + " SELECT 1 FROM nation " + " UNION ALL " + " SELECT 1 FROM nation))", LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); List<PlanNode> remotes = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll(); assertEquals(remotes.size(), 2, "There should be exactly two RemoteExchanges"); assertEquals(((ExchangeNode) remotes.get(0)).getType(), GATHER); assertEquals(((ExchangeNode) remotes.get(1)).getType(), REPARTITION); }
private void validateShowStatsSubquery(ShowStats node, Query query, QuerySpecification querySpecification, Plan plan) { // The following properties of SELECT subquery are required: // - only one relation in FROM // - only predicates that can be pushed down can be in the where clause // - no group by // - no having // - no set quantifier Optional<FilterNode> filterNode = searchFrom(plan.getRoot()) .where(FilterNode.class::isInstance) .findSingle(); check(!filterNode.isPresent(), node, "Only predicates that can be pushed down are supported in the SHOW STATS WHERE clause"); check(querySpecification.getFrom().isPresent(), node, "There must be exactly one table in query passed to SHOW STATS SELECT clause"); check(querySpecification.getFrom().get() instanceof Table, node, "There must be exactly one table in query passed to SHOW STATS SELECT clause"); check(!query.getWith().isPresent(), node, "WITH is not supported by SHOW STATS SELECT clause"); check(!querySpecification.getOrderBy().isPresent(), node, "ORDER BY is not supported in SHOW STATS SELECT clause"); check(!querySpecification.getLimit().isPresent(), node, "LIMIT is not supported by SHOW STATS SELECT clause"); check(!querySpecification.getHaving().isPresent(), node, "HAVING is not supported in SHOW STATS SELECT clause"); check(!querySpecification.getGroupBy().isPresent(), node, "GROUP BY is not supported in SHOW STATS SELECT clause"); check(!querySpecification.getSelect().isDistinct(), node, "DISTINCT is not supported by SHOW STATS SELECT clause"); List<SelectItem> selectItems = querySpecification.getSelect().getSelectItems(); check(selectItems.size() == 1 && selectItems.get(0) instanceof AllColumns, node, "Only SELECT * is supported in SHOW STATS SELECT clause"); }
private static void assertAtMostOneAggregationBetweenRemoteExchanges(Plan plan) { List<PlanNode> fragments = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteExchange) .findAll() .stream() .flatMap(exchangeNode -> exchangeNode.getSources().stream()) .collect(toList()); for (PlanNode fragment : fragments) { List<PlanNode> aggregations = searchFrom(fragment) .where(AggregationNode.class::isInstance) .recurseOnlyWhen(TestUnion::isNotRemoteExchange) .findAll(); assertFalse(aggregations.size() > 1, "More than a single AggregationNode between remote exchanges"); } }
private static int countOfMatchingNodes(Plan plan, Predicate<PlanNode> predicate) { return searchFrom(plan.getRoot()).where(predicate).count(); }
private String getPlanText(Session session, String sql) { return localQueryRunner.inTransaction(session, transactionSession -> { Plan plan = localQueryRunner.createPlan(transactionSession, sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, WarningCollector.NOOP); return PlanPrinter.textLogicalPlan( plan.getRoot(), plan.getTypes(), localQueryRunner.getMetadata().getFunctionRegistry(), plan.getStatsAndCosts(), transactionSession, 0, false); }); } }
public SubPlan createSubPlans(Session session, Plan plan, boolean forceSingleNode) { Fragmenter fragmenter = new Fragmenter(session, metadata, plan.getTypes(), plan.getStatsAndCosts()); FragmentProperties properties = new FragmentProperties(new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), plan.getRoot().getOutputSymbols())); if (forceSingleNode || isForceSingleNodeOutput(session)) { properties = properties.setSingleNodeDistribution(); } PlanNode root = SimplePlanRewriter.rewriteWith(fragmenter, plan.getRoot(), properties); SubPlan subPlan = fragmenter.buildRootFragment(root, properties); subPlan = reassignPartitioningHandleIfNecessary(session, subPlan); subPlan = analyzeGroupedExecution(session, subPlan); checkState(!isForceSingleNodeOutput(session) || subPlan.getFragment().getPartitioning().isSingleNode(), "Root of PlanFragment is not single node"); // TODO: Remove query_max_stage_count session property and use queryManagerConfig.getMaxStageCount() here sanityCheckFragmentedPlan(subPlan, getQueryMaxStageCount(session)); return subPlan; }
private void assertPlanContainsNoFilter(String sql) { assertFalse( searchFrom(plan(sql, LogicalPlanner.Stage.OPTIMIZED).getRoot()) .where(isInstanceOfAny(FilterNode.class)) .matches(), "Unexpected node for query: " + sql); } }
private void assertPlanContainsNoApplyOrAnyJoin(String sql) { assertFalse( searchFrom(plan(sql, LogicalPlanner.Stage.OPTIMIZED).getRoot()) .where(isInstanceOfAny(ApplyNode.class, JoinNode.class, IndexJoinNode.class, SemiJoinNode.class, LateralJoinNode.class)) .matches(), "Unexpected node for query: " + sql); }
public static void assertPlan(Session session, Metadata metadata, StatsProvider statsProvider, Plan actual, Lookup lookup, PlanMatchPattern pattern) { MatchResult matches = actual.getRoot().accept(new PlanMatchingVisitor(session, metadata, statsProvider, lookup), pattern); if (!matches.isMatch()) { String formattedPlan = textLogicalPlan(actual.getRoot(), actual.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); PlanNode resolvedPlan = resolveGroupReferences(actual.getRoot(), lookup); String resolvedFormattedPlan = textLogicalPlan(resolvedPlan, actual.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); throw new AssertionError(format( "Plan does not match, expected [\n\n%s\n] but found [\n\n%s\n] which resolves to [\n\n%s\n]", pattern, formattedPlan, resolvedFormattedPlan)); } } }
private static int countFinalAggregationNodes(Plan plan) { return searchFrom(plan.getRoot()) .where(node -> node instanceof AggregationNode && ((AggregationNode) node).getStep() == FINAL) .count(); }
private static int countSingleStreamingAggregations(Plan plan) { return searchFrom(plan.getRoot()) .where(node -> node instanceof AggregationNode && ((AggregationNode) node).getStep() == SINGLE && ((AggregationNode) node).isStreamable()) .count(); } }
private Consumer<Plan> assertRemoteExchangesCount(int expectedRemoteExchangesCount) { return plan -> { int actualRemoteExchangesCount = searchFrom(plan.getRoot()) .where(node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope() == ExchangeNode.Scope.REMOTE) .findAll() .size(); if (actualRemoteExchangesCount != expectedRemoteExchangesCount) { Session session = getSession(); Metadata metadata = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getMetadata(); String formattedPlan = textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); throw new AssertionError(format( "Expected [\n%s\n] remote exchanges but found [\n%s\n] remote exchanges. Actual plan is [\n\n%s\n]", expectedRemoteExchangesCount, actualRemoteExchangesCount, formattedPlan)); } }; }