/** * Creates a JSON representation of the given dataflow plan. * * @param plan The dataflow plan. * @return The dataflow plan (prior to optimization) as a JSON string. */ public static String getPlanAsJSON(Plan plan) { List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan); return new PlanJSONDumpGenerator().getPactPlanAsJSON(sinks); } }
/** * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned * and all channels have a shipping strategy assigned. * * For more details on the optimization phase, see the comments for * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}. * * @param program The program to be translated. * @return The optimized plan. * * @throws CompilerException * Thrown, if the plan is invalid or the optimizer encountered an inconsistent * situation during the compilation process. */ public OptimizedPlan compile(Plan program) throws CompilerException { final OptimizerPostPass postPasser = getPostPassFromPlan(program); return compile(program, postPasser); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { if (node instanceof SinkJoiner) { return new SinkJoinerPlanNode((SinkJoiner) node, in1, in2); } else { throw new CompilerException(); } }
private JobGraph getJobGraph(final Plan plan) { final Optimizer pc = new Optimizer(new DataStatistics(), getConfiguration()); final OptimizedPlan op = pc.compile(plan); final JobGraphGenerator jgg = new JobGraphGenerator(); return jgg.compileJobGraph(op); } }
@Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); OptimizedPlan optPlan = opt.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan); }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ @Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
private OptimizedPlan compileProgram(String jobName) { Plan p = createProgramPlan(jobName); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); return pc.compile(p); }
@Before public void setup() { Configuration flinkConf = new Configuration(); this.dataStats = new DataStatistics(); this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator(), flinkConf); this.withStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM); this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator(), flinkConf); this.noStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM); }
public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices, ActorSystemLoader actorSystemLoader) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = Preconditions.checkNotNull(actorSystemLoader); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
public OptimizedPlan compileWithStats(Plan p) { return this.withStatsCompiler.compile(p); }
@Override public void postVisit(OptimizerNode visitable) { visitable.identifyDynamicPath(this.costWeight); // check that there is no nested iteration on the dynamic path if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) { throw new CompilerException("Nested iterations are currently not supported."); } } }
private int getNumberOfSendersPerReceiver(DistributionPattern pattern, int numSenders, int numReceivers) { if (pattern == DistributionPattern.ALL_TO_ALL) { return numSenders; } else if (pattern == DistributionPattern.POINTWISE) { if (numSenders != numReceivers) { if (numReceivers == 1) { return numSenders; } else if (numSenders == 1) { return 1; } else { throw new CompilerException("Error: A changing parallelism is currently " + "not supported between tasks within an iteration."); } } else { return 1; } } else { throw new CompilerException("Unknown distribution pattern for channels: " + pattern); } }
public void setSourceStatistics(GenericDataSourceBase<?, ?> source, FileBaseStatistics stats) { final String key = CACHE_KEY + this.statCounter++; this.dataStats.cacheBaseStatistics(stats, key); source.setStatisticsKey(key); }
/** * Helpers to generate the JobGraph. */ private static JobGraph getJobGraph(Plan plan) { Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); JobGraphGenerator jgg = new JobGraphGenerator(); OptimizedPlan op = pc.compile(plan); return jgg.compileJobGraph(op); } }
/** * Creates a JSON representation of the given dataflow's execution plan. * * @param plan The dataflow plan. * @return The dataflow's execution plan, as a JSON string. * @throws Exception Thrown, if the optimization process that creates the execution plan failed. */ public static String optimizerPlanAsJSON(Plan plan) throws Exception { final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism(); Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); pc.setDefaultParallelism(parallelism); OptimizedPlan op = pc.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op); }
public OptimizedPlan compileNoStats(Plan p) { return this.noStatsCompiler.compile(p); }
public static boolean[] getDirections(Ordering o, int numFields) { final boolean[] dirs = o.getFieldSortDirections(); if (dirs.length == numFields) { return dirs; } else if (dirs.length > numFields) { final boolean[] subSet = new boolean[numFields]; System.arraycopy(dirs, 0, subSet, 0, numFields); return subSet; } else { throw new CompilerException(); } }
@Override public String getExecutionPlan() throws Exception { Plan plan = createProgramPlan(null, false); this.optimizerPlan = compiler.compile(plan); // do not go on with anything now! throw new ProgramAbortException(); }
/** * Creates a new node for the bulk iteration. * * @param iteration The bulk iteration the node represents. */ public BulkIterationNode(BulkIterationBase<?> iteration) { super(iteration); if (iteration.getMaximumNumberOfIterations() <= 0) { throw new CompilerException("BulkIteration must have a maximum number of iterations specified."); } int numIters = iteration.getMaximumNumberOfIterations(); this.costWeight = (numIters > 0 && numIters < OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) ? numIters : OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT; }
@Override public JobExecutionResult execute(String jobName) throws Exception { Plan plan = createProgramPlan(jobName); this.optimizerPlan = compiler.compile(plan); // do not go on with anything now! throw new ProgramAbortException(); }