/** * Creates a new optimizer instance that uses the statistics object to determine properties about the input. * Given those statistics, the optimizer can make better choices for the execution strategies. * * @param stats * The statistics to be used to determine the input properties. */ public Optimizer(DataStatistics stats, Configuration config) { this(stats, new DefaultCostEstimator(), config); }
/** * Creates a new optimizer instance. The optimizer has no access to statistics about the * inputs and can hence not determine any properties. It will perform all optimization with * unknown sizes and hence use only the heuristic cost functions, which result in the selection * of the most robust execution strategies. */ public Optimizer(Configuration config) { this(null, new DefaultCostEstimator(), config); }
public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices, ActorSystemLoader actorSystemLoader) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = Preconditions.checkNotNull(actorSystemLoader); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
@Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); OptimizedPlan optPlan = opt.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan); }
@Before public void setup() { Configuration flinkConf = new Configuration(); this.dataStats = new DataStatistics(); this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator(), flinkConf); this.withStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM); this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator(), flinkConf); this.noStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM); }
Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration); FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);
/** * Creates a instance that submits the programs to the JobManager defined in the * configuration. This method will try to resolve the JobManager hostname and throw an exception * if that is not possible. * * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer. * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down */ public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = new LazyActorSystemLoader( highAvailabilityServices, Time.milliseconds(lookupTimeout.toMillis()), flinkConfig, log); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
@Test public void testGetExecutionPlan() { try { PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); InetAddress mockAddress = InetAddress.getLocalHost(); InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345); Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName()); config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
JobID jobID) throws ProgramInvocationException { Thread.currentThread().setContextClassLoader(packagedProgram.getUserCodeClassLoader()); final Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration); final FlinkPlan flinkPlan;
@Test public void testGetExecutionPlan() { try { jobManagerSystem.actorOf( Props.create(FailureReturningActor.class), JobMaster.JOB_MANAGER_NAME); PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, 1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), c);
/** * Creates a new optimizer instance that uses the statistics object to determine properties about the input. * Given those statistics, the optimizer can make better choices for the execution strategies. * * @param stats * The statistics to be used to determine the input properties. */ public Optimizer(DataStatistics stats, Configuration config) { this(stats, new DefaultCostEstimator(), config); }
/** * Creates a new optimizer instance. The optimizer has no access to statistics about the * inputs and can hence not determine any properties. It will perform all optimization with * unknown sizes and hence use only the heuristic cost functions, which result in the selection * of the most robust execution strategies. */ public Optimizer(Configuration config) { this(null, new DefaultCostEstimator(), config); }
/** * Creates a new optimizer instance that uses the statistics object to determine properties about the input. * Given those statistics, the optimizer can make better choices for the execution strategies. * * @param stats * The statistics to be used to determine the input properties. */ public Optimizer(DataStatistics stats, Configuration config) { this(stats, new DefaultCostEstimator(), config); }
/** * Creates a new optimizer instance. The optimizer has no access to statistics about the * inputs and can hence not determine any properties. It will perform all optimization with * unknown sizes and hence use only the heuristic cost functions, which result in the selection * of the most robust execution strategies. */ public Optimizer(Configuration config) { this(null, new DefaultCostEstimator(), config); }
/** * Creates a new optimizer instance that uses the statistics object to determine properties about the input. * Given those statistics, the optimizer can make better choices for the execution strategies. * * @param stats * The statistics to be used to determine the input properties. */ public Optimizer(DataStatistics stats, Configuration config) { this(stats, new DefaultCostEstimator(), config); }
/** * Creates a new optimizer instance that uses the statistics object to determine properties about the input. * Given those statistics, the optimizer can make better choices for the execution strategies. * * @param stats * The statistics to be used to determine the input properties. */ public Optimizer(DataStatistics stats, Configuration config) { this(stats, new DefaultCostEstimator(), config); }
@Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); OptimizedPlan optPlan = opt.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan); }
@Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); OptimizedPlan optPlan = opt.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan); }
@Override public String getOptimizerPlanAsJSON(Plan plan) throws Exception { Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); OptimizedPlan optPlan = opt.compile(plan); return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan); }