@Override protected void optimizeOperatorPlan(ParseContext pCtx, Set<ReadEntity> inputs, Set<WriteEntity> outputs) throws SemanticException { PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE); OptimizeSparkProcContext procCtx = new OptimizeSparkProcContext(conf, pCtx, inputs, outputs); // Run Spark Dynamic Partition Pruning runDynamicPartitionPruning(procCtx); // Annotation OP tree with statistics runStatsAnnotation(procCtx); // Run Dynamic Partitioning sort Optimization. runDynPartitionSortOptimizations(procCtx); // Set reducer parallelism runSetReducerParallelism(procCtx); // Run Join releated optimizations runJoinOptimizations(procCtx); if(conf.isSparkDPPAny()){ // Remove DPP based on expected size of the output data runRemoveDynamicPruning(procCtx); // Remove cyclic dependencies for DPP runCycleAnalysisForPartitionPruning(procCtx); // Remove nested DPPs SparkUtilities.removeNestedDPP(procCtx); } // Re-run constant propagation so we fold any new constants introduced by the operator optimizers // Specifically necessary for DPP because we might have created lots of "and true and true" conditions if (procCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) { new ConstantPropagate(ConstantPropagateProcCtx.ConstantPropagateOption.SHORTCUT).transform(pCtx); } PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE); }