@Override protected PlanNodeStatsEstimate visitExpression(Expression node, Void context) { return PlanNodeStatsEstimate.unknown(); }
@Override public PlanNodeStatsEstimate calculateStats(PlanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) { Iterator<Rule<?>> ruleIterator = getCandidates(node).iterator(); while (ruleIterator.hasNext()) { Rule<?> rule = ruleIterator.next(); Optional<PlanNodeStatsEstimate> calculatedStats = calculateStats(rule, node, sourceStats, lookup, session, types); if (calculatedStats.isPresent()) { return calculatedStats.get(); } } return PlanNodeStatsEstimate.unknown(); }
public static PlanNodeStatsEstimate estimateExpressionToExpressionComparison( PlanNodeStatsEstimate inputStatistics, SymbolStatsEstimate leftExpressionStatistics, Optional<Symbol> leftExpressionSymbol, SymbolStatsEstimate rightExpressionStatistics, Optional<Symbol> rightExpressionSymbol, ComparisonExpression.Operator operator) { switch (operator) { case EQUAL: return estimateExpressionEqualToExpression(inputStatistics, leftExpressionStatistics, leftExpressionSymbol, rightExpressionStatistics, rightExpressionSymbol); case NOT_EQUAL: return estimateExpressionNotEqualToExpression(inputStatistics, leftExpressionStatistics, leftExpressionSymbol, rightExpressionStatistics, rightExpressionSymbol); case LESS_THAN: case LESS_THAN_OR_EQUAL: case GREATER_THAN: case GREATER_THAN_OR_EQUAL: case IS_DISTINCT_FROM: return PlanNodeStatsEstimate.unknown(); default: throw new IllegalArgumentException("Unexpected comparison operator: " + operator); } }
public StatsCalculatorAssertion(StatsCalculator statsCalculator, Session session, PlanNode planNode, TypeProvider types) { this.statsCalculator = requireNonNull(statsCalculator, "statsCalculator can not be null"); this.session = requireNonNull(session, "sesssion can not be null"); this.planNode = requireNonNull(planNode, "planNode is null"); this.types = requireNonNull(types, "types is null"); sourcesStats = new HashMap<>(); planNode.getSources().forEach(child -> sourcesStats.put(child, PlanNodeStatsEstimate.unknown())); }
private SymbolStatsAssertion assertCalculate(Expression scalarExpression) { return assertCalculate(scalarExpression, PlanNodeStatsEstimate.unknown()); }
private PlanNodeStatsEstimate estimateLogicalOr(Expression left, Expression right) { PlanNodeStatsEstimate leftEstimate = process(left); if (leftEstimate.isOutputRowCountUnknown()) { return PlanNodeStatsEstimate.unknown(); } PlanNodeStatsEstimate rightEstimate = process(right); if (rightEstimate.isOutputRowCountUnknown()) { return PlanNodeStatsEstimate.unknown(); } PlanNodeStatsEstimate andEstimate = new FilterExpressionStatsCalculatingVisitor(leftEstimate, session, types).process(right); if (andEstimate.isOutputRowCountUnknown()) { return PlanNodeStatsEstimate.unknown(); } return capStats( subtractSubsetStats( addStatsAndSumDistinctValues(leftEstimate, rightEstimate), andEstimate), input); }
public static PlanNodeStatsEstimate estimateExpressionToLiteralComparison( PlanNodeStatsEstimate inputStatistics, SymbolStatsEstimate expressionStatistics, Optional<Symbol> expressionSymbol, OptionalDouble literalValue, ComparisonExpression.Operator operator) { switch (operator) { case EQUAL: return estimateExpressionEqualToLiteral(inputStatistics, expressionStatistics, expressionSymbol, literalValue); case NOT_EQUAL: return estimateExpressionNotEqualToLiteral(inputStatistics, expressionStatistics, expressionSymbol, literalValue); case LESS_THAN: case LESS_THAN_OR_EQUAL: return estimateExpressionLessThanLiteral(inputStatistics, expressionStatistics, expressionSymbol, literalValue); case GREATER_THAN: case GREATER_THAN_OR_EQUAL: return estimateExpressionGreaterThanLiteral(inputStatistics, expressionStatistics, expressionSymbol, literalValue); case IS_DISTINCT_FROM: return PlanNodeStatsEstimate.unknown(); default: throw new IllegalArgumentException("Unexpected comparison operator: " + operator); } }
private boolean isPlanNodeStatsAndCostsUnknown(PlanNode node) { PlanNodeStatsEstimate stats = estimatedStatsAndCosts.getStats().getOrDefault(node.getId(), PlanNodeStatsEstimate.unknown()); PlanNodeCostEstimate cost = estimatedStatsAndCosts.getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); return stats.isOutputRowCountUnknown() || cost.equals(PlanNodeCostEstimate.unknown()); }
private String formatPlanNodeStatsAndCost(PlanNode node) { PlanNodeStatsEstimate stats = estimatedStatsAndCosts.getStats().getOrDefault(node.getId(), PlanNodeStatsEstimate.unknown()); PlanNodeCostEstimate cost = estimatedStatsAndCosts.getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); return format("{rows: %s (%s), cpu: %s, memory: %s, network: %s}", formatAsLong(stats.getOutputRowCount()), formatEstimateAsDataSize(stats.getOutputSizeInBytes(node.getOutputSymbols(), types)), formatDouble(cost.getCpuCost()), formatDouble(cost.getMemoryCost()), formatDouble(cost.getNetworkCost())); } }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(SpatialJoinNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) { PlanNodeStatsEstimate leftStats = sourceStats.getStats(node.getLeft()); PlanNodeStatsEstimate rightStats = sourceStats.getStats(node.getRight()); PlanNodeStatsEstimate crossJoinStats = crossJoinStats(node, leftStats, rightStats); switch (node.getType()) { case INNER: return Optional.of(statsCalculator.filterStats(crossJoinStats, node.getFilter(), session, types)); case LEFT: return Optional.of(PlanNodeStatsEstimate.unknown()); default: throw new IllegalArgumentException("Unknown spatial join type: " + node.getType()); } }
private static List<OptionalDouble> getEstimatedValuesInternal(List<Metric> metrics, String query, QueryRunner runner, Session session) // TODO inline back this method { Plan queryPlan = runner.createPlan(session, query, WarningCollector.NOOP); OutputNode outputNode = (OutputNode) queryPlan.getRoot(); PlanNodeStatsEstimate outputNodeStats = queryPlan.getStatsAndCosts().getStats().getOrDefault(queryPlan.getRoot().getId(), PlanNodeStatsEstimate.unknown()); StatsContext statsContext = buildStatsContext(queryPlan, outputNode); return getEstimatedValues(metrics, outputNodeStats, statsContext); }
private static PlanNodeStatsEstimate addStats(PlanNodeStatsEstimate left, PlanNodeStatsEstimate right, RangeAdditionStrategy strategy) { if (left.isOutputRowCountUnknown() || right.isOutputRowCountUnknown()) { return PlanNodeStatsEstimate.unknown(); } PlanNodeStatsEstimate.Builder statsBuilder = PlanNodeStatsEstimate.builder(); double newRowCount = left.getOutputRowCount() + right.getOutputRowCount(); concat(left.getSymbolsWithKnownStatistics().stream(), right.getSymbolsWithKnownStatistics().stream()) .distinct() .forEach(symbol -> { SymbolStatsEstimate symbolStats = SymbolStatsEstimate.zero(); if (newRowCount > 0) { symbolStats = addColumnStats( left.getSymbolStatistics(symbol), left.getOutputRowCount(), right.getSymbolStatistics(symbol), right.getOutputRowCount(), newRowCount, strategy); } statsBuilder.addSymbolStatistics(symbol, symbolStats); }); return statsBuilder.setOutputRowCount(newRowCount).build(); }
@Test public void testSubtractRowCount() { PlanNodeStatsEstimate unknownStats = statistics(NaN, NaN, NaN, StatisticRange.empty()); PlanNodeStatsEstimate first = statistics(40, NaN, NaN, StatisticRange.empty()); PlanNodeStatsEstimate second = statistics(10, NaN, NaN, StatisticRange.empty()); assertEquals(subtractSubsetStats(unknownStats, unknownStats), PlanNodeStatsEstimate.unknown()); assertEquals(subtractSubsetStats(first, unknownStats), PlanNodeStatsEstimate.unknown()); assertEquals(subtractSubsetStats(unknownStats, second), PlanNodeStatsEstimate.unknown()); assertEquals(subtractSubsetStats(first, second).getOutputRowCount(), 30.0); }
@Test public void testAddRowCount() { PlanNodeStatsEstimate unknownStats = statistics(NaN, NaN, NaN, StatisticRange.empty()); PlanNodeStatsEstimate first = statistics(10, NaN, NaN, StatisticRange.empty()); PlanNodeStatsEstimate second = statistics(20, NaN, NaN, StatisticRange.empty()); assertEquals(addStatsAndSumDistinctValues(unknownStats, unknownStats), PlanNodeStatsEstimate.unknown()); assertEquals(addStatsAndSumDistinctValues(first, unknownStats), PlanNodeStatsEstimate.unknown()); assertEquals(addStatsAndSumDistinctValues(unknownStats, second), PlanNodeStatsEstimate.unknown()); assertEquals(addStatsAndSumDistinctValues(first, second).getOutputRowCount(), 30.0); }
private void assertCostHasUnknownComponentsForUnknownStats(PlanNode node, Map<String, Type> types) { new CostAssertionBuilder(calculateCumulativeCost( costCalculatorUsingExchanges, node, planNode -> PlanNodeCostEstimate.unknown(), planNode -> PlanNodeStatsEstimate.unknown(), types)) .hasUnknownComponents(); new CostAssertionBuilder(calculateCumulativeCost( costCalculatorWithEstimatedExchanges, node, planNode -> PlanNodeCostEstimate.unknown(), planNode -> PlanNodeStatsEstimate.unknown(), types)) .hasUnknownComponents(); }
@Override protected PlanNodeStatsEstimate visitIsNotNullPredicate(IsNotNullPredicate node, Void context) { if (node.getValue() instanceof SymbolReference) { Symbol symbol = Symbol.from(node.getValue()); SymbolStatsEstimate symbolStats = input.getSymbolStatistics(symbol); PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.buildFrom(input); result.setOutputRowCount(input.getOutputRowCount() * (1 - symbolStats.getNullsFraction())); result.addSymbolStatistics(symbol, symbolStats.mapNullsFraction(x -> 0.0)); return result.build(); } return PlanNodeStatsEstimate.unknown(); }
@Test public void testDoesNotFireWithNoStats() { assertReorderJoins() .on(p -> p.join( INNER, p.values(new PlanNodeId("valuesA"), ImmutableList.of(p.symbol("A1")), TWO_ROWS), p.values(new PlanNodeId("valuesB"), p.symbol("B1")), ImmutableList.of(new EquiJoinClause(p.symbol("A1"), p.symbol("B1"))), ImmutableList.of(p.symbol("A1")), Optional.empty())) .overrideStats("valuesA", PlanNodeStatsEstimate.unknown()) .doesNotFire(); }
@Test public void testCastUnknown() { assertCalculate(new Cast(new SymbolReference("a"), "bigint"), PlanNodeStatsEstimate.unknown()) .lowValueUnknown() .highValueUnknown() .distinctValuesCountUnknown() .nullsFractionUnknown() .dataSizeUnknown(); }
@Test public void testLeftJoinMissingStats() { PlanNodeStatsEstimate leftStats = planNodeStats( 1, new SymbolStatistics(LEFT_JOIN_COLUMN, SymbolStatsEstimate.unknown()), new SymbolStatistics(LEFT_OTHER_COLUMN, SymbolStatsEstimate.unknown())); PlanNodeStatsEstimate rightStats = planNodeStats( 1, new SymbolStatistics(RIGHT_JOIN_COLUMN, SymbolStatsEstimate.unknown()), new SymbolStatistics(RIGHT_OTHER_COLUMN, SymbolStatsEstimate.unknown())); assertJoinStats(LEFT, leftStats, rightStats, PlanNodeStatsEstimate.unknown()); }
@Override protected PlanNodeStatsEstimate visitIsNullPredicate(IsNullPredicate node, Void context) { if (node.getValue() instanceof SymbolReference) { Symbol symbol = Symbol.from(node.getValue()); SymbolStatsEstimate symbolStats = input.getSymbolStatistics(symbol); PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.buildFrom(input); result.setOutputRowCount(input.getOutputRowCount() * symbolStats.getNullsFraction()); result.addSymbolStatistics(symbol, SymbolStatsEstimate.builder() .setNullsFraction(1.0) .setLowValue(NaN) .setHighValue(NaN) .setDistinctValuesCount(0.0) .build()); return result.build(); } return PlanNodeStatsEstimate.unknown(); }