private PlanNodeCostEstimate calculateCumulativeCost(PlanNode node) { PlanNodeCostEstimate localCosts = costCalculator.calculateCost(node, statsProvider, session, types); PlanNodeCostEstimate sourcesCost = node.getSources().stream() .map(this::getCumulativeCost) .reduce(PlanNodeCostEstimate.zero(), PlanNodeCostEstimate::add); PlanNodeCostEstimate cumulativeCost = localCosts.add(sourcesCost); return cumulativeCost; } }
@Override public PlanNodeCostEstimate calculateCost(PlanNode node, StatsProvider stats, Session session, TypeProvider types) { ExchangeCostEstimator exchangeCostEstimator = new ExchangeCostEstimator(stats, types, taskCountEstimator); PlanNodeCostEstimate estimatedExchangeCost = node.accept(exchangeCostEstimator, null); return costCalculator.calculateCost(node, stats, session, types).add(estimatedExchangeCost); }
private PlanNodeCostEstimate calculateJoinCost(PlanNode join, PlanNode probe, PlanNode build, boolean replicated) { PlanNodeCostEstimate joinInputCost = calculateJoinInputCost( probe, build, stats, types, replicated, taskCountEstimator.estimateSourceDistributedTaskCount()); PlanNodeCostEstimate joinOutputCost = calculateJoinOutputCost(join); return joinInputCost.add(joinOutputCost); }
public static PlanNodeCostEstimate calculateJoinCostWithoutOutput( PlanNode probe, PlanNode build, StatsProvider stats, TypeProvider types, boolean replicated, int estimatedSourceDistributedTaskCount) { PlanNodeCostEstimate exchangesCost = calculateJoinExchangeCost( probe, build, stats, types, replicated, estimatedSourceDistributedTaskCount); PlanNodeCostEstimate inputCost = calculateJoinInputCost( probe, build, stats, types, replicated, estimatedSourceDistributedTaskCount); return exchangesCost.add(inputCost); }
private PlanNodeCostEstimate calculateCumulativeCost(PlanNode node) { PlanNodeCostEstimate sourcesCost = node.getSources().stream() .map(this::getCumulativeCost) .reduce(PlanNodeCostEstimate.zero(), PlanNodeCostEstimate::add); return costCalculator.calculateCost(node, statsProvider, session, types).add(sourcesCost); } }
private static PlanNodeCostEstimate calculateJoinExchangeCost( PlanNode probe, PlanNode build, StatsProvider stats, TypeProvider types, boolean replicated, int estimatedSourceDistributedTaskCount) { double probeSizeInBytes = stats.getStats(probe).getOutputSizeInBytes(probe.getOutputSymbols(), types); double buildSizeInBytes = stats.getStats(build).getOutputSizeInBytes(build.getOutputSymbols(), types); if (replicated) { // assuming the probe side of a replicated join is always source distributed PlanNodeCostEstimate replicateCost = calculateRemoteReplicateCost(buildSizeInBytes, estimatedSourceDistributedTaskCount); // cost of the copies repartitioning is added in CostCalculatorUsingExchanges#calculateJoinCost PlanNodeCostEstimate localRepartitionCost = calculateLocalRepartitionCost(buildSizeInBytes); return replicateCost.add(localRepartitionCost); } else { PlanNodeCostEstimate probeCost = calculateRemoteRepartitionCost(probeSizeInBytes); PlanNodeCostEstimate buildRemoteRepartitionCost = calculateRemoteRepartitionCost(buildSizeInBytes); PlanNodeCostEstimate buildLocalRepartitionCost = calculateLocalRepartitionCost(buildSizeInBytes); return probeCost .add(buildRemoteRepartitionCost) .add(buildLocalRepartitionCost); } }
@Override public PlanNodeCostEstimate visitAggregation(AggregationNode node, Void context) { PlanNode source = node.getSource(); double inputSizeInBytes = getStats(source).getOutputSizeInBytes(source.getOutputSymbols(), types); PlanNodeCostEstimate remoteRepartitionCost = calculateRemoteRepartitionCost(inputSizeInBytes); PlanNodeCostEstimate localRepartitionCost = calculateLocalRepartitionCost(inputSizeInBytes); // TODO consider cost of aggregation itself, not only exchanges, based on aggregation's properties return remoteRepartitionCost.add(localRepartitionCost); }
private PlanNodeCostEstimate calculateCumulativeCost( CostCalculator costCalculator, PlanNode node, Function<PlanNode, PlanNodeCostEstimate> costs, Function<PlanNode, PlanNodeStatsEstimate> stats, Map<String, Type> types) { PlanNodeCostEstimate localCost = costCalculator.calculateCost( node, planNode -> requireNonNull(stats.apply(planNode), "no stats for node"), session, TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue)))); PlanNodeCostEstimate sourcesCost = node.getSources().stream() .map(source -> requireNonNull(costs.apply(source), format("no cost for source: %s", source.getId()))) .reduce(PlanNodeCostEstimate.zero(), PlanNodeCostEstimate::add); return sourcesCost.add(localCost); }
@Test public void testEvictCostOnReplace() { PlanNode y = node(); PlanNode x = node(y); Memo memo = new Memo(idAllocator, x); int xGroup = memo.getRootGroup(); int yGroup = getChildGroup(memo, memo.getRootGroup()); PlanNodeCostEstimate yCost = PlanNodeCostEstimate.cpuCost(42); PlanNodeCostEstimate xCost = yCost.add(PlanNodeCostEstimate.networkCost(37)); memo.storeCumulativeCost(yGroup, yCost); memo.storeCumulativeCost(xGroup, xCost); assertEquals(memo.getCumulativeCost(yGroup), Optional.of(yCost)); assertEquals(memo.getCumulativeCost(xGroup), Optional.of(xCost)); memo.replace(yGroup, node(), "rule"); assertEquals(memo.getCumulativeCost(yGroup), Optional.empty()); assertEquals(memo.getCumulativeCost(xGroup), Optional.empty()); }