public static PlanNodeCostEstimate networkCost(double networkCost) { return new PlanNodeCostEstimate(0, 0, networkCost); }
private PlanNodeCostEstimate calculateCumulativeCost(PlanNode node) { PlanNodeCostEstimate localCosts = costCalculator.calculateCost(node, statsProvider, session, types); PlanNodeCostEstimate sourcesCost = node.getSources().stream() .map(this::getCumulativeCost) .reduce(PlanNodeCostEstimate.zero(), PlanNodeCostEstimate::add); PlanNodeCostEstimate cumulativeCost = localCosts.add(sourcesCost); return cumulativeCost; } }
public static PlanNodeCostEstimate calculateLocalRepartitionCost(double inputSizeInBytes) { return cpuCost(inputSizeInBytes); }
private JoinEnumerationResult(Optional<PlanNode> planNode, PlanNodeCostEstimate cost) { this.planNode = requireNonNull(planNode, "planNode is null"); this.cost = requireNonNull(cost, "cost is null"); checkArgument((cost.hasUnknownComponents() || cost.equals(PlanNodeCostEstimate.infinite())) && !planNode.isPresent() || (!cost.hasUnknownComponents() || !cost.equals(PlanNodeCostEstimate.infinite())) && planNode.isPresent(), "planNode should be present if and only if cost is known"); }
public int compare(Session session, PlanNodeCostEstimate left, PlanNodeCostEstimate right) { requireNonNull(session, "session is null"); requireNonNull(left, "left is null"); requireNonNull(right, "right is null"); checkArgument(!left.hasUnknownComponents() && !right.hasUnknownComponents(), "cannot compare unknown costs"); double leftCost = left.getCpuCost() * cpuWeight + left.getMemoryCost() * memoryWeight + left.getNetworkCost() * networkWeight; double rightCost = right.getCpuCost() * cpuWeight + right.getMemoryCost() * memoryWeight + right.getNetworkCost() * networkWeight; return Double.compare(leftCost, rightCost); } }
private String formatPlanNodeStatsAndCost(PlanNode node) { PlanNodeStatsEstimate stats = estimatedStatsAndCosts.getStats().getOrDefault(node.getId(), PlanNodeStatsEstimate.unknown()); PlanNodeCostEstimate cost = estimatedStatsAndCosts.getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); return format("{rows: %s (%s), cpu: %s, memory: %s, network: %s}", formatAsLong(stats.getOutputRowCount()), formatEstimateAsDataSize(stats.getOutputSizeInBytes(node.getOutputSymbols(), types)), formatDouble(cost.getCpuCost()), formatDouble(cost.getMemoryCost()), formatDouble(cost.getNetworkCost())); } }
@Test public void testEvictCostOnReplace() { PlanNode y = node(); PlanNode x = node(y); Memo memo = new Memo(idAllocator, x); int xGroup = memo.getRootGroup(); int yGroup = getChildGroup(memo, memo.getRootGroup()); PlanNodeCostEstimate yCost = PlanNodeCostEstimate.cpuCost(42); PlanNodeCostEstimate xCost = yCost.add(PlanNodeCostEstimate.networkCost(37)); memo.storeCumulativeCost(yGroup, yCost); memo.storeCumulativeCost(xGroup, xCost); assertEquals(memo.getCumulativeCost(yGroup), Optional.of(yCost)); assertEquals(memo.getCumulativeCost(xGroup), Optional.of(xCost)); memo.replace(yGroup, node(), "rule"); assertEquals(memo.getCumulativeCost(yGroup), Optional.empty()); assertEquals(memo.getCumulativeCost(xGroup), Optional.empty()); }
@Test public void testUnknownCost() { CostComparator costComparator = new CostComparator(1.0, 1.0, 1.0); Session session = testSessionBuilder().build(); assertThrows(IllegalArgumentException.class, () -> costComparator.compare(session, PlanNodeCostEstimate.zero(), PlanNodeCostEstimate.unknown())); assertThrows(IllegalArgumentException.class, () -> costComparator.compare(session, PlanNodeCostEstimate.unknown(), PlanNodeCostEstimate.zero())); assertThrows(IllegalArgumentException.class, () -> costComparator.compare(session, PlanNodeCostEstimate.unknown(), PlanNodeCostEstimate.unknown())); }
@Override public PlanNodeCostEstimate calculateCost(PlanNode node, StatsProvider stats, Session session, TypeProvider types) { ExchangeCostEstimator exchangeCostEstimator = new ExchangeCostEstimator(stats, types, taskCountEstimator); PlanNodeCostEstimate estimatedExchangeCost = node.accept(exchangeCostEstimator, null); return costCalculator.calculateCost(node, stats, session, types).add(estimatedExchangeCost); }
@Override protected PlanNodeCostEstimate visitPlan(PlanNode node, Void context) { return PlanNodeCostEstimate.unknown(); }
@Override public PlanNodeCostEstimate visitAggregation(AggregationNode node, Void context) { if (node.getStep() != FINAL && node.getStep() != SINGLE) { return PlanNodeCostEstimate.unknown(); } PlanNodeStatsEstimate aggregationStats = getStats(node); PlanNodeStatsEstimate sourceStats = getStats(node.getSource()); double cpuCost = sourceStats.getOutputSizeInBytes(node.getSource().getOutputSymbols(), types); double memoryCost = aggregationStats.getOutputSizeInBytes(node.getOutputSymbols(), types); return new PlanNodeCostEstimate(cpuCost, memoryCost, 0); }
private boolean isPlanNodeStatsAndCostsUnknown(PlanNode node) { PlanNodeStatsEstimate stats = estimatedStatsAndCosts.getStats().getOrDefault(node.getId(), PlanNodeStatsEstimate.unknown()); PlanNodeCostEstimate cost = estimatedStatsAndCosts.getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); return stats.isOutputRowCountUnknown() || cost.equals(PlanNodeCostEstimate.unknown()); }
CostAssertionBuilder hasUnknownComponents() { assertTrue(actual.hasUnknownComponents()); return this; } }
public static PlanNodeCostEstimate calculateRemoteGatherCost(double inputSizeInBytes) { return networkCost(inputSizeInBytes); }
CostAssertionBuilder network(double value) { assertEquals(actual.getNetworkCost(), value, 0.000001); return this; }
CostAssertionBuilder memory(double value) { assertEquals(actual.getMemoryCost(), value, 0.000001); return this; }
CostAssertionBuilder cpu(double value) { assertEquals(actual.getCpuCost(), value, 0.000001); return this; }
@Test public void testDoesNotCreateJoinWhenPartitionedOnCrossJoin() { PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator(); PlanBuilder p = new PlanBuilder(idAllocator, queryRunner.getMetadata()); Symbol a1 = p.symbol("A1"); Symbol b1 = p.symbol("B1"); MultiJoinNode multiJoinNode = new MultiJoinNode( new LinkedHashSet<>(ImmutableList.of(p.values(a1), p.values(b1))), TRUE_LITERAL, ImmutableList.of(a1, b1)); JoinEnumerator joinEnumerator = new JoinEnumerator( new CostComparator(1, 1, 1), multiJoinNode.getFilter(), createContext()); JoinEnumerationResult actual = joinEnumerator.createJoinAccordingToPartitioning(multiJoinNode.getSources(), multiJoinNode.getOutputSymbols(), ImmutableSet.of(0)); assertFalse(actual.getPlanNode().isPresent()); assertEquals(actual.getCost(), PlanNodeCostEstimate.infinite()); }
static JoinEnumerationResult createJoinEnumerationResult(Optional<PlanNode> planNode, PlanNodeCostEstimate cost) { if (cost.hasUnknownComponents()) { return UNKNOWN_COST_RESULT; } if (cost.equals(PlanNodeCostEstimate.infinite())) { return INFINITE_COST_RESULT; } return new JoinEnumerationResult(planNode, cost); } }
private PlanNodeCostEstimate calculateJoinCost(PlanNode join, PlanNode probe, PlanNode build, boolean replicated) { PlanNodeCostEstimate joinInputCost = calculateJoinInputCost( probe, build, stats, types, replicated, taskCountEstimator.estimateSourceDistributedTaskCount()); PlanNodeCostEstimate joinOutputCost = calculateJoinOutputCost(join); return joinInputCost.add(joinOutputCost); }