/** * This function performs only the first step to the compilation process - the creation of the optimizer * representation of the plan. No estimations or enumerations of alternatives are done here. * * @param program The plan to generate the optimizer representation for. * @return The optimizer representation of the plan, as a collection of all data sinks * from the plan can be traversed. */ public static List<DataSinkNode> createPreOptimizedPlan(Plan program) { GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null); program.accept(graphCreator); return graphCreator.getSinks(); }
GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode); program.accept(graphCreator); if (graphCreator.getSinks().size() == 1) { rootNode = graphCreator.getSinks().get(0); else if (graphCreator.getSinks().size() > 1) { Iterator<DataSinkNode> iter = graphCreator.getSinks().iterator(); rootNode = iter.next(); rootNode.accept(new IdAndEstimatesVisitor(this.statistics)); UnionParallelismAndForwardEnforcer unionEnforcer = new UnionParallelismAndForwardEnforcer(); rootNode.accept(unionEnforcer); BranchesVisitor branchingVisitor = new BranchesVisitor(); rootNode.accept(branchingVisitor); InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator); rootNode.accept(propsVisitor); OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program); plan.accept(new BinaryUnionReplacer()); plan.accept(new RangePartitionRewriter(plan));
private static List<DataSinkNode> convertPlan(Plan p) { GraphCreatingVisitor dagCreator = new GraphCreatingVisitor(17, p.getExecutionConfig().getExecutionMode()); // create the DAG p.accept(dagCreator); List<DataSinkNode> sinks = dagCreator.getSinks(); // build a single root and run the branch tracking logic OptimizerNode rootNode; if (sinks.size() == 1) { rootNode = sinks.get(0); } else { Iterator<DataSinkNode> iter = sinks.iterator(); rootNode = iter.next(); while (iter.hasNext()) { rootNode = new SinkJoiner(rootNode, iter.next()); } } rootNode.accept(new IdAndEstimatesVisitor(null)); rootNode.accept(new BranchesVisitor()); return sinks; } }
final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true, iterNode.getParallelism(), defaultDataExchangeMode, closure); StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight()); iterNode.acceptForStepFunction(identifier); StepFunctionValidator wsf = new StepFunctionValidator(); iter.getNextWorkset().accept(wsf); if (!wsf.hasFoundWorkset()) { throw new CompilerException("In the given program, the next workset does not depend on the workset. " + "This is a prerequisite in delta iterations."); final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor( this, true, iterNode.getParallelism(), defaultDataExchangeMode, closure); StaticDynamicPathIdentifier pathIdentifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight()); iterNode.acceptForStepFunction(pathIdentifier);
@Override public void postVisit(PlanNode node) { if(node instanceof IterationPlanNode) { IterationPlanNode iNode = (IterationPlanNode)node; if(!visitedIterationNodes.contains(iNode)) { visitedIterationNodes.add(iNode); iNode.acceptForStepFunction(this); } } final Iterable<Channel> inputChannels = node.getInputs(); for (Channel channel : inputChannels) { ShipStrategyType shipStrategy = channel.getShipStrategy(); // Make sure we only optimize the DAG for range partition, and do not optimize multi times. if (shipStrategy == ShipStrategyType.PARTITION_RANGE) { if(channel.getDataDistribution() == null) { if (node.isOnDynamicPath()) { throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution."); } PlanNode channelSource = channel.getSource(); List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel); channelSource.getOutgoingChannels().remove(channel); channelSource.getOutgoingChannels().addAll(newSourceOutputChannels); } } } }
this.terminationCriterion.accept(new InterestingPropertyVisitor(estimator)); this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator)); this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode); program.accept(graphCreator); if (graphCreator.getSinks().size() == 1) { rootNode = graphCreator.getSinks().get(0); else if (graphCreator.getSinks().size() > 1) { Iterator<DataSinkNode> iter = graphCreator.getSinks().iterator(); rootNode = iter.next(); rootNode.accept(new IdAndEstimatesVisitor(this.statistics)); UnionParallelismAndForwardEnforcer unionEnforcer = new UnionParallelismAndForwardEnforcer(); rootNode.accept(unionEnforcer); BranchesVisitor branchingVisitor = new BranchesVisitor(); rootNode.accept(branchingVisitor); InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator); rootNode.accept(propsVisitor); OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program); plan.accept(new BinaryUnionReplacer()); plan.accept(new RangePartitionRewriter(plan));
/** * This function performs only the first step to the compilation process - the creation of the optimizer * representation of the plan. No estimations or enumerations of alternatives are done here. * * @param program The plan to generate the optimizer representation for. * @return The optimizer representation of the plan, as a collection of all data sinks * from the plan can be traversed. */ public static List<DataSinkNode> createPreOptimizedPlan(Plan program) { GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null); program.accept(graphCreator); return graphCreator.getSinks(); }
@Override public void postVisit(PlanNode node) { if(node instanceof IterationPlanNode) { IterationPlanNode iNode = (IterationPlanNode)node; if(!visitedIterationNodes.contains(iNode)) { visitedIterationNodes.add(iNode); iNode.acceptForStepFunction(this); } } final Iterable<Channel> inputChannels = node.getInputs(); for (Channel channel : inputChannels) { ShipStrategyType shipStrategy = channel.getShipStrategy(); // Make sure we only optimize the DAG for range partition, and do not optimize multi times. if (shipStrategy == ShipStrategyType.PARTITION_RANGE) { if(channel.getDataDistribution() == null) { if (node.isOnDynamicPath()) { throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution."); } PlanNode channelSource = channel.getSource(); List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel); channelSource.getOutgoingChannels().remove(channel); channelSource.getOutgoingChannels().addAll(newSourceOutputChannels); } } } }
this.solutionSetDeltaRootConnection.setInterestingProperties(partitionedIP.clone()); InterestingPropertyVisitor ipv = new InterestingPropertyVisitor(estimator); this.nextWorkset.accept(ipv); this.solutionSetDelta.accept(ipv);
GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode); program.accept(graphCreator); if (graphCreator.getSinks().size() == 1) { rootNode = graphCreator.getSinks().get(0); else if (graphCreator.getSinks().size() > 1) { Iterator<DataSinkNode> iter = graphCreator.getSinks().iterator(); rootNode = iter.next(); rootNode.accept(new IdAndEstimatesVisitor(this.statistics)); UnionParallelismAndForwardEnforcer unionEnforcer = new UnionParallelismAndForwardEnforcer(); rootNode.accept(unionEnforcer); BranchesVisitor branchingVisitor = new BranchesVisitor(); rootNode.accept(branchingVisitor); InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator); rootNode.accept(propsVisitor); OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program); plan.accept(new BinaryUnionReplacer()); plan.accept(new RangePartitionRewriter(plan));
/** * This function performs only the first step to the compilation process - the creation of the optimizer * representation of the plan. No estimations or enumerations of alternatives are done here. * * @param program The plan to generate the optimizer representation for. * @return The optimizer representation of the plan, as a collection of all data sinks * from the plan can be traversed. */ public static List<DataSinkNode> createPreOptimizedPlan(Plan program) { GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null); program.accept(graphCreator); return graphCreator.getSinks(); }
GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode); program.accept(graphCreator); if (graphCreator.getSinks().size() == 1) { rootNode = graphCreator.getSinks().get(0); else if (graphCreator.getSinks().size() > 1) { Iterator<DataSinkNode> iter = graphCreator.getSinks().iterator(); rootNode = iter.next(); rootNode.accept(new IdAndEstimatesVisitor(this.statistics)); UnionParallelismAndForwardEnforcer unionEnforcer = new UnionParallelismAndForwardEnforcer(); rootNode.accept(unionEnforcer); BranchesVisitor branchingVisitor = new BranchesVisitor(); rootNode.accept(branchingVisitor); InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator); rootNode.accept(propsVisitor); OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program); plan.accept(new BinaryUnionReplacer()); plan.accept(new RangePartitionRewriter(plan));
/** * This function performs only the first step to the compilation process - the creation of the optimizer * representation of the plan. No estimations or enumerations of alternatives are done here. * * @param program The plan to generate the optimizer representation for. * @return The optimizer representation of the plan, as a collection of all data sinks * from the plan can be traversed. */ public static List<DataSinkNode> createPreOptimizedPlan(Plan program) { GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null); program.accept(graphCreator); return graphCreator.getSinks(); }
GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode); program.accept(graphCreator); if (graphCreator.getSinks().size() == 1) { rootNode = graphCreator.getSinks().get(0); else if (graphCreator.getSinks().size() > 1) { Iterator<DataSinkNode> iter = graphCreator.getSinks().iterator(); rootNode = iter.next(); rootNode.accept(new IdAndEstimatesVisitor(this.statistics)); BranchesVisitor branchingVisitor = new BranchesVisitor(); rootNode.accept(branchingVisitor); InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator); rootNode.accept(propsVisitor); OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program); plan.accept(new BinaryUnionReplacer()); plan.accept(new RangePartitionRewriter(plan));
/** * This function performs only the first step to the compilation process - the creation of the optimizer * representation of the plan. No estimations or enumerations of alternatives are done here. * * @param program The plan to generate the optimizer representation for. * @return The optimizer representation of the plan, as a collection of all data sinks * from the plan can be traversed. */ public static List<DataSinkNode> createPreOptimizedPlan(Plan program) { GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null); program.accept(graphCreator); return graphCreator.getSinks(); }