/** * Flattens the subplan and returns all PlanFragments in the tree */ public List<PlanFragment> getAllFragments() { ImmutableList.Builder<PlanFragment> fragments = ImmutableList.builder(); fragments.add(getFragment()); for (SubPlan child : getChildren()) { fragments.addAll(child.getAllFragments()); } return fragments.build(); }
private SubPlan reassignPartitioningHandleIfNecessary(Session session, SubPlan subPlan) { return reassignPartitioningHandleIfNecessaryHelper(session, subPlan, subPlan.getFragment().getPartitioning()); }
private StageExecutionPlan doPlan(SubPlan root, Session session, ImmutableList.Builder<SplitSource> allSplitSources) { PlanFragment currentFragment = root.getFragment(); // get splits for this fragment, this is lazy so split assignments aren't actually calculated here Map<PlanNodeId, SplitSource> splitSources = currentFragment.getRoot().accept(new Visitor(session, currentFragment.getStageExecutionDescriptor(), allSplitSources), null); // create child stages ImmutableList.Builder<StageExecutionPlan> dependencies = ImmutableList.builder(); for (SubPlan childPlan : root.getChildren()) { dependencies.add(doPlan(childPlan, session, allSplitSources)); } return new StageExecutionPlan( currentFragment, splitSources, dependencies.build()); }
private static void printSubPlan(SubPlan plan, Map<PlanFragmentId, PlanFragment> fragmentsById, PlanNodeIdGenerator idGenerator, StringBuilder output) { PlanFragment fragment = plan.getFragment(); printFragmentNodes(output, fragment, idGenerator); fragment.getRoot().accept(new EdgePrinter(output, fragmentsById, idGenerator), null); for (SubPlan child : plan.getChildren()) { printSubPlan(child, fragmentsById, idGenerator, output); } }
private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan) { PlanFragment fragment = subPlan.getFragment(); GroupedExecutionProperties properties = fragment.getRoot().accept(new GroupedExecutionTagger(session, metadata, nodePartitioningManager), null); if (properties.isSubTreeUseful()) { boolean preferDynamic = fragment.getRemoteSourceNodes().isEmpty() && isDynamicSchduleForGroupedExecution(session); BucketNodeMap bucketNodeMap = nodePartitioningManager.getBucketNodeMap(session, fragment.getPartitioning(), preferDynamic); if (bucketNodeMap.isDynamic()) { fragment = fragment.withDynamicLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes()); } else { fragment = fragment.withFixedLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes()); } } ImmutableList.Builder<SubPlan> result = ImmutableList.builder(); for (SubPlan child : subPlan.getChildren()) { result.add(analyzeGroupedExecution(session, child)); } return new SubPlan(fragment, result.build()); }
StageExecutionDescriptor stageExecutionDescriptor = subplan.getFragment().getStageExecutionDescriptor(); LocalExecutionPlan localExecutionPlan = executionPlanner.plan( taskContext, stageExecutionDescriptor, subplan.getFragment().getRoot(), subplan.getFragment().getPartitioningScheme().getOutputLayout(), plan.getTypes(), subplan.getFragment().getPartitionedSources(), outputFactory); for (TableScanNode tableScan : findTableScanNodes(subplan.getFragment().getRoot())) { TableLayoutHandle layout = tableScan.getLayout().get(); ImmutableSet<PlanNodeId> partitionedSources = ImmutableSet.copyOf(subplan.getFragment().getPartitionedSources()); for (TaskSource source : sources) { DriverFactory driverFactory = driverFactoriesBySource.get(source.getPlanNodeId());
private PlanNodeCostEstimate calculateCumulativeCostFragmentedPlan(PlanNode node, StatsCalculator statsCalculator, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider); CostProvider costProvider = new CachingCostProvider(costCalculatorUsingExchanges, statsProvider, Optional.empty(), session, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); }
private CostAssertionBuilder assertCostFragmentedPlan( PlanNode node, Map<String, PlanNodeCostEstimate> costs, Map<String, PlanNodeStatsEstimate> stats, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator(stats), session, typeProvider); CostProvider costProvider = new TestingCostProvider(costs, costCalculatorUsingExchanges, statsProvider, session, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return new CostAssertionBuilder(subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown())); }
public SubPlan createSubPlans(Session session, Plan plan, boolean forceSingleNode) { Fragmenter fragmenter = new Fragmenter(session, metadata, plan.getTypes(), plan.getStatsAndCosts()); FragmentProperties properties = new FragmentProperties(new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), plan.getRoot().getOutputSymbols())); if (forceSingleNode || isForceSingleNodeOutput(session)) { properties = properties.setSingleNodeDistribution(); } PlanNode root = SimplePlanRewriter.rewriteWith(fragmenter, plan.getRoot(), properties); SubPlan subPlan = fragmenter.buildRootFragment(root, properties); subPlan = reassignPartitioningHandleIfNecessary(session, subPlan); subPlan = analyzeGroupedExecution(session, subPlan); checkState(!isForceSingleNodeOutput(session) || subPlan.getFragment().getPartitioning().isSingleNode(), "Root of PlanFragment is not single node"); // TODO: Remove query_max_stage_count session property and use queryManagerConfig.getMaxStageCount() here sanityCheckFragmentedPlan(subPlan, getQueryMaxStageCount(session)); return subPlan; }
private SubPlan reassignPartitioningHandleIfNecessaryHelper(Session session, SubPlan subPlan, PartitioningHandle newOutputPartitioningHandle) PlanFragment fragment = subPlan.getFragment();
PartitioningHandle partitioningHandle = plan.getRoot().getFragment().getPartitioningScheme().getPartitioning().getHandle(); OutputBuffers rootOutputBuffers = createInitialEmptyOutputBuffers(partitioningHandle) .withBuffer(OUTPUT_BUFFER_ID, BROADCAST_PARTITION_ID)
/** * Flattens the subplan and returns all PlanFragments in the tree */ public List<PlanFragment> getAllFragments() { ImmutableList.Builder<PlanFragment> fragments = ImmutableList.builder(); fragments.add(getFragment()); for (SubPlan child : getChildren()) { fragments.addAll(child.getAllFragments()); } return fragments.build(); }
public StageExecutionPlan plan(SubPlan root, Session session) { PlanFragment currentFragment = root.getFragment(); // get splits for this fragment, this is lazy so split assignments aren't actually calculated here Visitor visitor = new Visitor(session); Optional<SplitSource> splits = currentFragment.getRoot().accept(visitor, null); // create child stages ImmutableList.Builder<StageExecutionPlan> dependencies = ImmutableList.builder(); for (SubPlan childPlan : root.getChildren()) { dependencies.add(plan(childPlan, session)); } return new StageExecutionPlan( currentFragment, splits, dependencies.build()); }
private static void printSubPlan(SubPlan plan, Map<PlanFragmentId, PlanFragment> fragmentsById, PlanNodeIdGenerator idGenerator, StringBuilder output) { PlanFragment fragment = plan.getFragment(); printFragmentNodes(output, fragment, idGenerator); fragment.getRoot().accept(new EdgePrinter(output, fragmentsById, idGenerator), null); for (SubPlan child : plan.getChildren()) { printSubPlan(child, fragmentsById, idGenerator, output); } }
subplan.getFragment().getRoot(), subplan.getFragment().getPartitionFunction().getOutputLayout(), plan.getTypes(), outputFactory, for (TableScanNode tableScan : findTableScanNodes(subplan.getFragment().getRoot())) { TableLayoutHandle layout = tableScan.getLayout().get();