public String getGraphvizPlan(Session session, Statement statement, Type planType, List<Expression> parameters, WarningCollector warningCollector) { DataDefinitionTask<?> task = dataDefinitionTask.get(statement.getClass()); if (task != null) { // todo format as graphviz return explainTask(statement, task, parameters); } switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); return PlanPrinter.graphvizLogicalPlan(plan.getRoot(), plan.getTypes()); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); return PlanPrinter.graphvizDistributedPlan(subPlan); } throw new IllegalArgumentException("Unhandled plan type: " + planType); }
public Plan plan(Analysis analysis, Stage stage) { PlanNode root = planStatement(analysis, analysis.getStatement()); planSanityChecker.validateIntermediatePlan(root, session, metadata, sqlParser, symbolAllocator.getTypes(), warningCollector); if (stage.ordinal() >= Stage.OPTIMIZED.ordinal()) { for (PlanOptimizer optimizer : planOptimizers) { root = optimizer.optimize(root, session, symbolAllocator.getTypes(), symbolAllocator, idAllocator, warningCollector); requireNonNull(root, format("%s returned a null plan", optimizer.getClass().getName())); } } if (stage.ordinal() >= Stage.OPTIMIZED_AND_VALIDATED.ordinal()) { // make sure we produce a valid plan after optimizations run. This is mainly to catch programming errors planSanityChecker.validateFinalPlan(root, session, metadata, sqlParser, symbolAllocator.getTypes(), warningCollector); } TypeProvider types = symbolAllocator.getTypes(); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session, types); return new Plan(root, types, StatsAndCosts.create(root, statsProvider, costProvider)); }
private String getPlanText(Session session, String sql) { return localQueryRunner.inTransaction(session, transactionSession -> { Plan plan = localQueryRunner.createPlan(transactionSession, sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, WarningCollector.NOOP); return PlanPrinter.textLogicalPlan( plan.getRoot(), plan.getTypes(), localQueryRunner.getMetadata().getFunctionRegistry(), plan.getStatsAndCosts(), transactionSession, 0, false); }); } }
private static List<OptionalDouble> getEstimatedValuesInternal(List<Metric> metrics, String query, QueryRunner runner, Session session) // TODO inline back this method { Plan queryPlan = runner.createPlan(session, query, WarningCollector.NOOP); OutputNode outputNode = (OutputNode) queryPlan.getRoot(); PlanNodeStatsEstimate outputNodeStats = queryPlan.getStatsAndCosts().getStats().getOrDefault(queryPlan.getRoot().getId(), PlanNodeStatsEstimate.unknown()); StatsContext statsContext = buildStatsContext(queryPlan, outputNode); return getEstimatedValues(metrics, outputNodeStats, statsContext); }
public String getJsonPlan(Session session, Statement statement, Type planType, List<Expression> parameters, WarningCollector warningCollector) { DataDefinitionTask<?> task = dataDefinitionTask.get(statement.getClass()); if (task != null) { // todo format as json return explainTask(statement, task, parameters); } switch (planType) { case IO: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); return textIOPlan(plan.getRoot(), metadata, session); default: throw new PrestoException(NOT_SUPPORTED, format("Unsupported explain plan type %s for JSON format", planType)); } }
public SubPlan createSubPlans(Plan plan) { Fragmenter fragmenter = new Fragmenter(plan.getSymbolAllocator().getTypes()); FragmentProperties properties = new FragmentProperties(new PartitionFunctionBinding(SINGLE_DISTRIBUTION, plan.getRoot().getOutputSymbols(), ImmutableList.of())) .setSingleNodeDistribution(); PlanNode root = SimplePlanRewriter.rewriteWith(fragmenter, plan.getRoot(), properties); SubPlan result = fragmenter.buildRootFragment(root, properties); result.sanityCheck(); return result; }
private static StatsContext buildStatsContext(Plan queryPlan, OutputNode outputNode) { ImmutableMap.Builder<String, Symbol> columnSymbols = ImmutableMap.builder(); for (int columnId = 0; columnId < outputNode.getColumnNames().size(); ++columnId) { columnSymbols.put(outputNode.getColumnNames().get(columnId), outputNode.getOutputSymbols().get(columnId)); } return new StatsContext(columnSymbols.build(), queryPlan.getTypes()); }
private Constraint<ColumnHandle> getConstraint(Plan plan) { Optional<TableScanNode> scanNode = searchFrom(plan.getRoot()) .where(TableScanNode.class::isInstance) .findSingle(); if (!scanNode.isPresent()) { return Constraint.alwaysFalse(); } return new Constraint<>(scanNode.get().getCurrentConstraint()); }
private static List<OptionalDouble> getEstimatedValuesInternal(List<Metric> metrics, String query, QueryRunner runner, Session session) // TODO inline back this method { Plan queryPlan = runner.createPlan(session, query, WarningCollector.NOOP); OutputNode outputNode = (OutputNode) queryPlan.getRoot(); PlanNodeStatsEstimate outputNodeStats = queryPlan.getStatsAndCosts().getStats().getOrDefault(queryPlan.getRoot().getId(), PlanNodeStatsEstimate.unknown()); StatsContext statsContext = buildStatsContext(queryPlan, outputNode); return getEstimatedValues(metrics, outputNodeStats, statsContext); }
public static void assertPlan(Session session, Metadata metadata, StatsCalculator statsCalculator, Plan actual, Lookup lookup, PlanMatchPattern pattern) { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, actual.getTypes()); assertPlan(session, metadata, statsProvider, actual, lookup, pattern); }
public static void assertPlan(Session session, Metadata metadata, StatsProvider statsProvider, Plan actual, Lookup lookup, PlanMatchPattern pattern) { MatchResult matches = actual.getRoot().accept(new PlanMatchingVisitor(session, metadata, statsProvider, lookup), pattern); if (!matches.isMatch()) { String formattedPlan = textLogicalPlan(actual.getRoot(), actual.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); PlanNode resolvedPlan = resolveGroupReferences(actual.getRoot(), lookup); String resolvedFormattedPlan = textLogicalPlan(resolvedPlan, actual.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); throw new AssertionError(format( "Plan does not match, expected [\n\n%s\n] but found [\n\n%s\n] which resolves to [\n\n%s\n]", pattern, formattedPlan, resolvedFormattedPlan)); } } }
public String getPlan(Session session, Statement statement, Type planType, List<Expression> parameters, WarningCollector warningCollector) { DataDefinitionTask<?> task = dataDefinitionTask.get(statement.getClass()); if (task != null) { return explainTask(statement, task, parameters); } switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); return PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionRegistry(), plan.getStatsAndCosts(), session, 0, false); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector); return PlanPrinter.textDistributedPlan(subPlan, metadata.getFunctionRegistry(), session, false); case IO: return IOPlanPrinter.textIOPlan(getLogicalPlan(session, statement, parameters, warningCollector).getRoot(), metadata, session); } throw new IllegalArgumentException("Unhandled plan type: " + planType); }
private String generateQueryPlan(String query) { String sql = query.replaceAll("\\s+;\\s+$", "") .replace("${database}.${schema}.", "") .replace("\"${database}\".\"${schema}\".\"${prefix}", "\""); Plan plan = plan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, false); JoinOrderPrinter joinOrderPrinter = new JoinOrderPrinter(); plan.getRoot().accept(joinOrderPrinter, 0); return joinOrderPrinter.result(); }
private static StatsContext buildStatsContext(Plan queryPlan, OutputNode outputNode) { ImmutableMap.Builder<String, Symbol> columnSymbols = ImmutableMap.builder(); for (int columnId = 0; columnId < outputNode.getColumnNames().size(); ++columnId) { columnSymbols.put(outputNode.getColumnNames().get(columnId), outputNode.getOutputSymbols().get(columnId)); } return new StatsContext(columnSymbols.build(), queryPlan.getTypes()); }
private PlanNodeCostEstimate calculateCumulativeCostFragmentedPlan(PlanNode node, StatsCalculator statsCalculator, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider); CostProvider costProvider = new CachingCostProvider(costCalculatorUsingExchanges, statsProvider, Optional.empty(), session, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); }
private Consumer<Plan> assertRemoteExchangesCount(int expectedRemoteExchangesCount) { return plan -> { int actualRemoteExchangesCount = searchFrom(plan.getRoot()) .where(node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope() == ExchangeNode.Scope.REMOTE) .findAll() .size(); if (actualRemoteExchangesCount != expectedRemoteExchangesCount) { Session session = getSession(); Metadata metadata = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getMetadata(); String formattedPlan = textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); throw new AssertionError(format( "Expected [\n%s\n] remote exchanges but found [\n%s\n] remote exchanges. Actual plan is [\n\n%s\n]", expectedRemoteExchangesCount, actualRemoteExchangesCount, formattedPlan)); } }; }
public SubPlan createSubPlans(Session session, Plan plan, boolean forceSingleNode) { Fragmenter fragmenter = new Fragmenter(session, metadata, plan.getTypes(), plan.getStatsAndCosts()); FragmentProperties properties = new FragmentProperties(new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), plan.getRoot().getOutputSymbols())); if (forceSingleNode || isForceSingleNodeOutput(session)) { properties = properties.setSingleNodeDistribution(); } PlanNode root = SimplePlanRewriter.rewriteWith(fragmenter, plan.getRoot(), properties); SubPlan subPlan = fragmenter.buildRootFragment(root, properties); subPlan = reassignPartitioningHandleIfNecessary(session, subPlan); subPlan = analyzeGroupedExecution(session, subPlan); checkState(!isForceSingleNodeOutput(session) || subPlan.getFragment().getPartitioning().isSingleNode(), "Root of PlanFragment is not single node"); // TODO: Remove query_max_stage_count session property and use queryManagerConfig.getMaxStageCount() here sanityCheckFragmentedPlan(subPlan, getQueryMaxStageCount(session)); return subPlan; }
private void assertPlanIsFullyDistributed(Plan plan) { int numberOfGathers = searchFrom(plan.getRoot()) .where(TestUnion::isRemoteGatheringExchange) .findAll() .size(); if (numberOfGathers == 0) { // there are no "gather" nodes, so the plan is expected to be fully distributed return; } assertTrue( searchFrom(plan.getRoot()) .recurseOnlyWhen(TestUnion::isNotRemoteGatheringExchange) .findAll() .stream() .noneMatch(this::shouldBeDistributed), "There is a node that should be distributed between output and first REMOTE GATHER ExchangeNode"); assertEquals(numberOfGathers, 1, "Only a single REMOTE GATHER was expected"); }
assertPlan(session, metadata, ruleApplication.statsProvider, new Plan(actual, types, StatsAndCosts.empty()), ruleApplication.lookup, pattern); return null; });
public String getPlan(Session session, Statement statement, Type planType) { DataDefinitionTask<?> task = dataDefinitionTask.get(statement.getClass()); if (task != null) { return explainTask(statement, task); } switch (planType) { case LOGICAL: Plan plan = getLogicalPlan(session, statement); return PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata, session); case DISTRIBUTED: SubPlan subPlan = getDistributedPlan(session, statement); return PlanPrinter.textDistributedPlan(subPlan, metadata, session); } throw new IllegalArgumentException("Unhandled plan type: " + planType); }