public OptimizedPlan compileWithStats(Plan p) { return this.withStatsCompiler.compile(p); }
public OptimizedPlan compileNoStats(Plan p) { return this.noStatsCompiler.compile(p); }
/** * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned * and all channels have a shipping strategy assigned. * * For more details on the optimization phase, see the comments for * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}. * * @param program The program to be translated. * @return The optimized plan. * * @throws CompilerException * Thrown, if the plan is invalid or the optimizer encountered an inconsistent * situation during the compilation process. */ public OptimizedPlan compile(Plan program) throws CompilerException { final OptimizerPostPass postPasser = getPostPassFromPlan(program); return compile(program, postPasser); }
@Override public JobExecutionResult execute(String jobName) throws Exception { Plan plan = createProgramPlan(jobName); this.optimizerPlan = compiler.compile(plan); // do not go on with anything now! throw new ProgramAbortException(); }
@Override public String getExecutionPlan() throws Exception { Plan plan = createProgramPlan(null, false); this.optimizerPlan = compiler.compile(plan); // do not go on with anything now! throw new ProgramAbortException(); }
public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException { Logger log = LoggerFactory.getLogger(ClusterClient.class); if (parallelism > 0 && p.getDefaultParallelism() <= 0) { log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism); p.setDefaultParallelism(parallelism); } log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism()); return compiler.compile(p); }
private OptimizedPlan compileProgram(String jobName) { Plan p = createProgramPlan(jobName); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); return pc.compile(p); }
@Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); OptimizedPlan optPlan = opt.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ public static String optimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
private JobGraph getJobGraph(final Plan plan) { final Optimizer pc = new Optimizer(new DataStatistics(), getConfiguration()); final OptimizedPlan op = pc.compile(plan); final JobGraphGenerator jgg = new JobGraphGenerator(); return jgg.compileJobGraph(op); } }
/** * Helpers to generate the JobGraph. */ private static JobGraph getJobGraph(Plan plan) { Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); JobGraphGenerator jgg = new JobGraphGenerator(); OptimizedPlan op = pc.compile(plan); return jgg.compileJobGraph(op); } }
OptimizedPlan op = pc.compile(plan);
flinkPlan = optimizer.compile(jobWithJars.getPlan()); } else if (packagedProgram.isUsingInteractiveMode()) { final OptimizerPlanEnvironment optimizerPlanEnvironment = new OptimizerPlanEnvironment(optimizer);
@Override public JobExecutionResult execute(String jobName) throws Exception { Plan plan = createProgramPlan(jobName); Optimizer pc = new Optimizer(new Configuration()); OptimizedPlan op = pc.compile(plan); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jobGraph = jgg.compileJobGraph(op); String jsonPlan = JsonPlanGenerator.generatePlan(jobGraph); // first check that the JSON is valid JsonParser parser = new JsonFactory().createJsonParser(jsonPlan); while (parser.nextToken() != null) {} validator.validateJson(jsonPlan); throw new AbortError(); }
OptimizedPlan op = pc.compile(plan);
OptimizedPlan op = pc.compile(plan);
@Test public void testTempInIterationTest() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class); DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 1, 0); DataSet<Tuple2<Long, Long>> update = iteration.getWorkset() .join(iteration.getSolutionSet()).where(0).equalTo(0) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); iteration.closeWith(update, update) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jg = jgg.compileJobGraph(oPlan); boolean solutionSetUpdateChecked = false; for(JobVertex v : jg.getVertices()) { if(v.getName().equals("SolutionSet Delta")) { // check if input of solution set delta is temped TaskConfig tc = new TaskConfig(v.getConfiguration()); assertTrue(tc.isInputAsynchronouslyMaterialized(0)); solutionSetUpdateChecked = true; } } assertTrue(solutionSetUpdateChecked); }
@Override public String getExecutionPlan() throws Exception { Plan plan = createProgramPlan(null, false); this.optimizerPlan = compiler.compile(plan); // do not go on with anything now! throw new ProgramAbortException(); }
@Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); OptimizedPlan optPlan = opt.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan); }