public void doesNotFire() { RuleApplication ruleApplication = applyRule(); if (ruleApplication.wasRuleApplied()) { fail(String.format( "Expected %s to not fire for:\n%s", rule.getClass().getName(), inTransaction(session -> textLogicalPlan(plan, ruleApplication.types, metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 2)))); } }
private PlanNodeCostEstimate calculateCumulativeCostFragmentedPlan(PlanNode node, StatsCalculator statsCalculator, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider); CostProvider costProvider = new CachingCostProvider(costCalculatorUsingExchanges, statsProvider, Optional.empty(), session, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); }
private boolean isPlanNodeStatsAndCostsUnknown(PlanNode node) { PlanNodeStatsEstimate stats = estimatedStatsAndCosts.getStats().getOrDefault(node.getId(), PlanNodeStatsEstimate.unknown()); PlanNodeCostEstimate cost = estimatedStatsAndCosts.getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); return stats.isOutputRowCountUnknown() || cost.equals(PlanNodeCostEstimate.unknown()); }
public Plan plan(Analysis analysis, Stage stage) { PlanNode root = planStatement(analysis, analysis.getStatement()); planSanityChecker.validateIntermediatePlan(root, session, metadata, sqlParser, symbolAllocator.getTypes(), warningCollector); if (stage.ordinal() >= Stage.OPTIMIZED.ordinal()) { for (PlanOptimizer optimizer : planOptimizers) { root = optimizer.optimize(root, session, symbolAllocator.getTypes(), symbolAllocator, idAllocator, warningCollector); requireNonNull(root, format("%s returned a null plan", optimizer.getClass().getName())); } } if (stage.ordinal() >= Stage.OPTIMIZED_AND_VALIDATED.ordinal()) { // make sure we produce a valid plan after optimizations run. This is mainly to catch programming errors planSanityChecker.validateFinalPlan(root, session, metadata, sqlParser, symbolAllocator.getTypes(), warningCollector); } TypeProvider types = symbolAllocator.getTypes(); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session, types); return new Plan(root, types, StatsAndCosts.create(root, statsProvider, costProvider)); }
private static List<OptionalDouble> getEstimatedValuesInternal(List<Metric> metrics, String query, QueryRunner runner, Session session) // TODO inline back this method { Plan queryPlan = runner.createPlan(session, query, WarningCollector.NOOP); OutputNode outputNode = (OutputNode) queryPlan.getRoot(); PlanNodeStatsEstimate outputNodeStats = queryPlan.getStatsAndCosts().getStats().getOrDefault(queryPlan.getRoot().getId(), PlanNodeStatsEstimate.unknown()); StatsContext statsContext = buildStatsContext(queryPlan, outputNode); return getEstimatedValues(metrics, outputNodeStats, statsContext); }
public static StatsAndCosts create(PlanNode root, StatsProvider statsProvider, CostProvider costProvider) { Iterable<PlanNode> planIterator = Traverser.forTree(PlanNode::getSources) .depthFirstPreOrder(root); ImmutableMap.Builder<PlanNodeId, PlanNodeStatsEstimate> stats = ImmutableMap.builder(); ImmutableMap.Builder<PlanNodeId, PlanNodeCostEstimate> costs = ImmutableMap.builder(); for (PlanNode node : planIterator) { stats.put(node.getId(), statsProvider.getStats(node)); costs.put(node.getId(), costProvider.getCumulativeCost(node)); } return new StatsAndCosts(stats.build(), costs.build()); } }
private SubPlan buildFragment(PlanNode root, FragmentProperties properties, PlanFragmentId fragmentId) { Set<Symbol> dependencies = SymbolsExtractor.extractOutputSymbols(root); List<PlanNodeId> schedulingOrder = scheduleOrder(root); boolean equals = properties.getPartitionedSources().equals(ImmutableSet.copyOf(schedulingOrder)); checkArgument(equals, "Expected scheduling order (%s) to contain an entry for all partitioned sources (%s)", schedulingOrder, properties.getPartitionedSources()); PlanFragment fragment = new PlanFragment( fragmentId, root, Maps.filterKeys(types.allTypes(), in(dependencies)), properties.getPartitioningHandle(), schedulingOrder, properties.getPartitioningScheme(), StageExecutionDescriptor.ungroupedExecution(), statsAndCosts.getForSubplan(root)); return new SubPlan(fragment, properties.getChildren()); }
private String formatPlan(PlanNode plan, TypeProvider types) { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, session, types); return inTransaction(session -> textLogicalPlan(plan, types, metadata.getFunctionRegistry(), StatsAndCosts.create(plan, statsProvider, costProvider), session, 2, false)); }
private static List<OptionalDouble> getEstimatedValuesInternal(List<Metric> metrics, String query, QueryRunner runner, Session session) // TODO inline back this method { Plan queryPlan = runner.createPlan(session, query, WarningCollector.NOOP); OutputNode outputNode = (OutputNode) queryPlan.getRoot(); PlanNodeStatsEstimate outputNodeStats = queryPlan.getStatsAndCosts().getStats().getOrDefault(queryPlan.getRoot().getId(), PlanNodeStatsEstimate.unknown()); StatsContext statsContext = buildStatsContext(queryPlan, outputNode); return getEstimatedValues(metrics, outputNodeStats, statsContext); }
public StatsAndCosts getForSubplan(PlanNode root) { Iterable<PlanNode> planIterator = Traverser.forTree(PlanNode::getSources) .depthFirstPreOrder(root); ImmutableMap.Builder<PlanNodeId, PlanNodeStatsEstimate> filteredStats = ImmutableMap.builder(); ImmutableMap.Builder<PlanNodeId, PlanNodeCostEstimate> filteredCosts = ImmutableMap.builder(); for (PlanNode node : planIterator) { if (stats.containsKey(node.getId())) { filteredStats.put(node.getId(), stats.get(node.getId())); } if (costs.containsKey(node.getId())) { filteredCosts.put(node.getId(), costs.get(node.getId())); } } return new StatsAndCosts(filteredStats.build(), filteredCosts.build()); }
public static String graphvizLogicalPlan(PlanNode plan, TypeProvider types) { PlanFragment fragment = new PlanFragment( new PlanFragmentId("graphviz_plan"), plan, types.allTypes(), SINGLE_DISTRIBUTION, ImmutableList.of(plan.getId()), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), plan.getOutputSymbols()), StageExecutionDescriptor.ungroupedExecution(), StatsAndCosts.empty()); return GraphvizPrinter.printLogical(ImmutableList.of(fragment)); }
private String formatPlanNodeStatsAndCost(PlanNode node) { PlanNodeStatsEstimate stats = estimatedStatsAndCosts.getStats().getOrDefault(node.getId(), PlanNodeStatsEstimate.unknown()); PlanNodeCostEstimate cost = estimatedStatsAndCosts.getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); return format("{rows: %s (%s), cpu: %s, memory: %s, network: %s}", formatAsLong(stats.getOutputRowCount()), formatEstimateAsDataSize(stats.getOutputSizeInBytes(node.getOutputSymbols(), types)), formatDouble(cost.getCpuCost()), formatDouble(cost.getMemoryCost()), formatDouble(cost.getNetworkCost())); } }
private CostAssertionBuilder assertCostFragmentedPlan( PlanNode node, Map<String, PlanNodeCostEstimate> costs, Map<String, PlanNodeStatsEstimate> stats, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator(stats), session, typeProvider); CostProvider costProvider = new TestingCostProvider(costs, costCalculatorUsingExchanges, statsProvider, session, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return new CostAssertionBuilder(subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown())); }
assertPlan(session, metadata, ruleApplication.statsProvider, new Plan(actual, types, StatsAndCosts.empty()), ruleApplication.lookup, pattern); return null; });
private static PlanFragment createValuesPlan() { Symbol symbol = new Symbol("column"); PlanNodeId valuesNodeId = new PlanNodeId("plan"); PlanFragment planFragment = new PlanFragment( new PlanFragmentId("plan"), new ValuesNode(valuesNodeId, ImmutableList.of(symbol), ImmutableList.of(ImmutableList.of(new StringLiteral("foo")))), ImmutableMap.of(symbol, VARCHAR), SOURCE_DISTRIBUTION, ImmutableList.of(valuesNodeId), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(symbol)), StageExecutionDescriptor.ungroupedExecution(), StatsAndCosts.empty()); return planFragment; } }
private static PlanFragment createFragment(PlanNode planNode) { ImmutableMap.Builder<Symbol, Type> types = ImmutableMap.builder(); for (Symbol symbol : planNode.getOutputSymbols()) { types.put(symbol, VARCHAR); } return new PlanFragment( new PlanFragmentId(planNode.getId() + "_fragment_id"), planNode, types.build(), SOURCE_DISTRIBUTION, ImmutableList.of(planNode.getId()), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), planNode.getOutputSymbols()), StageExecutionDescriptor.ungroupedExecution(), StatsAndCosts.empty()); } }
public static void assertPlan(Session session, Metadata metadata, StatsProvider statsProvider, Plan actual, Lookup lookup, PlanMatchPattern pattern) { MatchResult matches = actual.getRoot().accept(new PlanMatchingVisitor(session, metadata, statsProvider, lookup), pattern); if (!matches.isMatch()) { String formattedPlan = textLogicalPlan(actual.getRoot(), actual.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); PlanNode resolvedPlan = resolveGroupReferences(actual.getRoot(), lookup); String resolvedFormattedPlan = textLogicalPlan(resolvedPlan, actual.getTypes(), metadata.getFunctionRegistry(), StatsAndCosts.empty(), session, 0); throw new AssertionError(format( "Plan does not match, expected [\n\n%s\n] but found [\n\n%s\n] which resolves to [\n\n%s\n]", pattern, formattedPlan, resolvedFormattedPlan)); } } }
private static PlanFragment createExchangePlanFragment() { PlanNode planNode = new RemoteSourceNode( new PlanNodeId("exchange"), ImmutableList.of(new PlanFragmentId("source")), ImmutableList.of(new Symbol("column")), Optional.empty(), REPARTITION); ImmutableMap.Builder<Symbol, Type> types = ImmutableMap.builder(); for (Symbol symbol : planNode.getOutputSymbols()) { types.put(symbol, VARCHAR); } return new PlanFragment( new PlanFragmentId("exchange_fragment_id"), planNode, types.build(), SOURCE_DISTRIBUTION, ImmutableList.of(planNode.getId()), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), planNode.getOutputSymbols()), StageExecutionDescriptor.ungroupedExecution(), StatsAndCosts.empty()); } }
new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(symbol)), StageExecutionDescriptor.ungroupedExecution(), StatsAndCosts.empty());
public MockRemoteTask createTableScanTask(TaskId taskId, Node newNode, List<Split> splits, PartitionedSplitCountTracker partitionedSplitCountTracker) { Symbol symbol = new Symbol("column"); PlanNodeId sourceId = new PlanNodeId("sourceId"); PlanFragment testFragment = new PlanFragment( new PlanFragmentId("test"), new TableScanNode( sourceId, new TableHandle(new ConnectorId("test"), new TestingTableHandle()), ImmutableList.of(symbol), ImmutableMap.of(symbol, new TestingColumnHandle("column"))), ImmutableMap.of(symbol, VARCHAR), SOURCE_DISTRIBUTION, ImmutableList.of(sourceId), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(symbol)), StageExecutionDescriptor.ungroupedExecution(), StatsAndCosts.empty()); ImmutableMultimap.Builder<PlanNodeId, Split> initialSplits = ImmutableMultimap.builder(); for (Split sourceSplit : splits) { initialSplits.put(sourceId, sourceSplit); } return createRemoteTask(TEST_SESSION, taskId, newNode, testFragment, initialSplits.build(), OptionalInt.empty(), createInitialEmptyOutputBuffers(BROADCAST), partitionedSplitCountTracker, true); }