OperatorUtils.removeBranch(nestedDPP);
if (desc.getTargetInfos().isEmpty()) { OperatorUtils.removeBranch(pruningSinkOp);
private void removeDPPOperator(Set<Operator<?>> component, OptimizeSparkProcContext context) { SparkPartitionPruningSinkOperator toRemove = null; for (Operator<?> o : component) { if (o instanceof SparkPartitionPruningSinkOperator) { // we want to remove the DPP with bigger data size if (toRemove == null || o.getConf().getStatistics().getDataSize() > toRemove.getConf().getStatistics() .getDataSize()) { toRemove = (SparkPartitionPruningSinkOperator) o; } } } if (toRemove == null) { return; } OperatorUtils.removeBranch(toRemove); // at this point we've found the fork in the op pipeline that has the pruning as a child plan. LOG.info("Disabling dynamic pruning for: " + toRemove.getConf().getTableScanNames() + ". Needed to break cyclic dependency"); }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procContext, Object... nodeOutputs) throws SemanticException { OptimizeSparkProcContext context = (OptimizeSparkProcContext) procContext; SparkPartitionPruningSinkOperator op = (SparkPartitionPruningSinkOperator) nd; SparkPartitionPruningSinkDesc desc = op.getConf(); if (desc.getStatistics().getDataSize() > context.getConf() .getLongVar(ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE)) { OperatorUtils.removeBranch(op); // at this point we've found the fork in the op pipeline that has the pruning as a child plan. LOG.info("Disabling dynamic pruning for: " + desc.getTableScan().getName() + ". Expected data size is too big: " + desc.getStatistics().getDataSize()); } return false; } }
OperatorUtils.removeBranch(partitionPruningSinkOp);
private void removeDPPOperator(Set<Operator<?>> component, OptimizeSparkProcContext context) { SparkPartitionPruningSinkOperator toRemove = null; for (Operator<?> o : component) { if (o instanceof SparkPartitionPruningSinkOperator) { // we want to remove the DPP with bigger data size if (toRemove == null || o.getConf().getStatistics().getDataSize() > toRemove.getConf().getStatistics() .getDataSize()) { toRemove = (SparkPartitionPruningSinkOperator) o; } } } if (toRemove == null) { return; } OperatorUtils.removeBranch(toRemove); // at this point we've found the fork in the op pipeline that has the pruning as a child plan. LOG.info("Disabling dynamic pruning for: " + toRemove.getConf().getTableScan().toString() + ". Needed to break cyclic dependency"); }
OperatorUtils.removeBranch(op);