private SubPlan getDistributedPlan(Session session, Statement statement, List<Expression> parameters, WarningCollector warningCollector) { Plan plan = getLogicalPlan(session, statement, parameters, warningCollector); return planFragmenter.createSubPlans(session, plan, false); } }
public SubPlan createSubPlans(Session session, Plan plan, boolean forceSingleNode) { Fragmenter fragmenter = new Fragmenter(session, metadata, plan.getTypes(), plan.getStatsAndCosts()); FragmentProperties properties = new FragmentProperties(new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), plan.getRoot().getOutputSymbols())); if (forceSingleNode || isForceSingleNodeOutput(session)) { properties = properties.setSingleNodeDistribution(); } PlanNode root = SimplePlanRewriter.rewriteWith(fragmenter, plan.getRoot(), properties); SubPlan subPlan = fragmenter.buildRootFragment(root, properties); subPlan = reassignPartitioningHandleIfNecessary(session, subPlan); subPlan = analyzeGroupedExecution(session, subPlan); checkState(!isForceSingleNodeOutput(session) || subPlan.getFragment().getPartitioning().isSingleNode(), "Root of PlanFragment is not single node"); // TODO: Remove query_max_stage_count session property and use queryManagerConfig.getMaxStageCount() here sanityCheckFragmentedPlan(subPlan, getQueryMaxStageCount(session)); return subPlan; }
private SubPlan reassignPartitioningHandleIfNecessary(Session session, SubPlan subPlan) { return reassignPartitioningHandleIfNecessaryHelper(session, subPlan, subPlan.getFragment().getPartitioning()); }
private SubPlan getDistributedPlan(Session session, Statement statement) { // analyze statement Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.of(this), experimentalSyntaxEnabled); Analysis analysis = analyzer.analyze(statement); PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator(); // plan statement LogicalPlanner logicalPlanner = new LogicalPlanner(session, planOptimizers, idAllocator, metadata); Plan plan = logicalPlanner.plan(analysis); return new PlanFragmenter().createSubPlans(plan); } }
private QueryExplainer getQueryExplainer() { Metadata metadata = queryRunner.getMetadata(); FeaturesConfig featuresConfig = new FeaturesConfig().setOptimizeHashGeneration(true); boolean forceSingleNode = queryRunner.getNodeCount() == 1; TaskCountEstimator taskCountEstimator = new TaskCountEstimator(queryRunner::getNodeCount); CostCalculator costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator); List<PlanOptimizer> optimizers = new PlanOptimizers( metadata, sqlParser, featuresConfig, forceSingleNode, new MBeanExporter(new TestingMBeanServer()), queryRunner.getSplitManager(), queryRunner.getPageSourceManager(), queryRunner.getStatsCalculator(), costCalculator, new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator), new CostComparator(featuresConfig), taskCountEstimator).get(); return new QueryExplainer( optimizers, new PlanFragmenter(metadata, queryRunner.getNodePartitioningManager(), new QueryManagerConfig()), metadata, queryRunner.getAccessControl(), sqlParser, queryRunner.getStatsCalculator(), costCalculator, ImmutableMap.of()); }
private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan) { PlanFragment fragment = subPlan.getFragment(); GroupedExecutionProperties properties = fragment.getRoot().accept(new GroupedExecutionTagger(session, metadata, nodePartitioningManager), null); if (properties.isSubTreeUseful()) { boolean preferDynamic = fragment.getRemoteSourceNodes().isEmpty() && isDynamicSchduleForGroupedExecution(session); BucketNodeMap bucketNodeMap = nodePartitioningManager.getBucketNodeMap(session, fragment.getPartitioning(), preferDynamic); if (bucketNodeMap.isDynamic()) { fragment = fragment.withDynamicLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes()); } else { fragment = fragment.withFixedLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes()); } } ImmutableList.Builder<SubPlan> result = ImmutableList.builder(); for (SubPlan child : subPlan.getChildren()) { result.add(analyzeGroupedExecution(session, child)); } return new SubPlan(fragment, result.build()); }
private SubPlan doAnalyzeQuery() { // time analysis phase long analysisStart = System.nanoTime(); // analyze query Analyzer analyzer = new Analyzer(stateMachine.getSession(), metadata, sqlParser, accessControl, Optional.of(queryExplainer), experimentalSyntaxEnabled); Analysis analysis = analyzer.analyze(statement); stateMachine.setUpdateType(analysis.getUpdateType()); // plan query PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator(); LogicalPlanner logicalPlanner = new LogicalPlanner(stateMachine.getSession(), planOptimizers, idAllocator, metadata); Plan plan = logicalPlanner.plan(analysis); // extract inputs List<Input> inputs = new InputExtractor(metadata, stateMachine.getSession()).extract(plan.getRoot()); stateMachine.setInputs(inputs); // fragment the plan SubPlan subplan = new PlanFragmenter().createSubPlans(plan); // record analysis time stateMachine.recordAnalysisTime(analysisStart); return subplan; }
new ColumnPropertyManager(), transactionManager); this.planFragmenter = new PlanFragmenter(this.metadata, this.nodePartitioningManager, new QueryManagerConfig()); this.joinCompiler = new JoinCompiler(metadata, featuresConfig); this.pageIndexerFactory = new GroupByHashPageIndexerFactory(joinCompiler);
SubPlan subplan = new PlanFragmenter().createSubPlans(plan); if (!subplan.getChildren().isEmpty()) { throw new AssertionError("Expected subplan to have no children");
@BeforeClass public void setUp() { TaskCountEstimator taskCountEstimator = new TaskCountEstimator(() -> NUMBER_OF_NODES); costCalculatorUsingExchanges = new CostCalculatorUsingExchanges(taskCountEstimator); costCalculatorWithEstimatedExchanges = new CostCalculatorWithEstimatedExchanges(costCalculatorUsingExchanges, taskCountEstimator); session = testSessionBuilder().setCatalog("tpch").build(); CatalogManager catalogManager = new CatalogManager(); catalogManager.registerCatalog(createBogusTestingCatalog("tpch")); transactionManager = createTestTransactionManager(catalogManager); metadata = createTestMetadataManager(transactionManager, new FeaturesConfig()); finalizerService = new FinalizerService(); finalizerService.start(); nodeScheduler = new NodeScheduler( new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)); nodePartitioningManager = new NodePartitioningManager(nodeScheduler); planFragmenter = new PlanFragmenter(metadata, nodePartitioningManager, new QueryManagerConfig()); }
private SubPlan fragment(Plan plan) { return inTransaction(session -> planFragmenter.createSubPlans(session, plan, false)); }
childrenBuilder.add(reassignPartitioningHandleIfNecessaryHelper(session, child, fragment.getPartitioning()));
private QueryExplainer getQueryExplainer() { Metadata metadata = queryRunner.getMetadata(); FeaturesConfig featuresConfig = new FeaturesConfig().setOptimizeHashGeneration(true); boolean forceSingleNode = queryRunner.getNodeCount() == 1; TaskCountEstimator taskCountEstimator = new TaskCountEstimator(queryRunner::getNodeCount); CostCalculator costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator); List<PlanOptimizer> optimizers = new PlanOptimizers( metadata, sqlParser, featuresConfig, forceSingleNode, new MBeanExporter(new TestingMBeanServer()), queryRunner.getSplitManager(), queryRunner.getPageSourceManager(), queryRunner.getStatsCalculator(), costCalculator, new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator), new CostComparator(featuresConfig), taskCountEstimator).get(); return new QueryExplainer( optimizers, new PlanFragmenter(metadata, queryRunner.getNodePartitioningManager(), new QueryManagerConfig()), metadata, queryRunner.getAccessControl(), sqlParser, queryRunner.getStatsCalculator(), costCalculator, ImmutableMap.of()); }
SubPlan subplan = planFragmenter.createSubPlans(session, plan, true); if (!subplan.getChildren().isEmpty()) { throw new AssertionError("Expected subplan to have no children");
private PlanRoot doAnalyzeQuery() { // time analysis phase stateMachine.beginAnalysis(); // plan query PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator(); LogicalPlanner logicalPlanner = new LogicalPlanner(stateMachine.getSession(), planOptimizers, idAllocator, metadata, sqlParser, statsCalculator, costCalculator, stateMachine.getWarningCollector()); Plan plan = logicalPlanner.plan(analysis); queryPlan.set(plan); // extract inputs List<Input> inputs = new InputExtractor(metadata, stateMachine.getSession()).extractInputs(plan.getRoot()); stateMachine.setInputs(inputs); // extract output Optional<Output> output = new OutputExtractor().extractOutput(plan.getRoot()); stateMachine.setOutput(output); // fragment the plan SubPlan fragmentedPlan = planFragmenter.createSubPlans(stateMachine.getSession(), plan, false); // record analysis time stateMachine.endAnalysis(); boolean explainAnalyze = analysis.getStatement() instanceof Explain && ((Explain) analysis.getStatement()).isAnalyze(); return new PlanRoot(fragmentedPlan, !explainAnalyze, extractConnectors(analysis)); }