protected StreamPlanEnvironment(ExecutionEnvironment env) { super(); this.env = env; int parallelism = env.getParallelism(); if (parallelism > 0) { setParallelism(parallelism); } else { // determine parallelism setParallelism(GlobalConfiguration.loadConfiguration().getInteger(CoreOptions.DEFAULT_PARALLELISM)); } }
Plan plan = translator.translateToPlan(this.sinks, jobName); if (getParallelism() > 0) { plan.setDefaultParallelism(getParallelism());
private void receiveParameters(ExecutionEnvironment env) throws IOException { for (int x = 0; x < Parameters.values().length; x++) { Tuple value = (Tuple) streamer.getRecord(true); switch (Parameters.valueOf(((String) value.getField(0)).toUpperCase())) { case DOP: Integer dop = value.<Integer>getField(1); env.setParallelism(dop); break; case RETRY: int retry = value.<Integer>getField(1); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(retry, 10000L)); break; case ID: currentEnvironmentID = value.<Integer>getField(1); break; } } if (env.getParallelism() < 0) { env.setParallelism(1); } }
@Test public void testForcedRebalancing() throws Exception { /* * Test forced rebalancing */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // generate some number in parallel DataSet<Long> ds = env.generateSequence(1, 3000); DataSet<Tuple2<Integer, Integer>> uniqLongs = ds // introduce some partition skew by filtering .filter(new Filter1()) // rebalance .rebalance() // count values in each partition .map(new PartitionIndexMapper()) .groupBy(0) .reduce(new Reducer1()) // round counts to mitigate runtime scheduling effects (lazy split assignment) .map(new Mapper1()); List<Tuple2<Integer, Integer>> result = uniqLongs.collect(); StringBuilder expected = new StringBuilder(); int numPerPartition = 2220 / env.getParallelism() / 10; for (int i = 0; i < env.getParallelism(); i++) { expected.append('(').append(i).append(',') .append(numPerPartition).append(")\n"); } compareResultAsText(result, expected.toString()); }
@Test public void testCountElementsPerPartition() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); long expectedSize = 100L; DataSet<Long> numbers = env.generateSequence(0, expectedSize - 1); DataSet<Tuple2<Integer, Long>> ds = DataSetUtils.countElementsPerPartition(numbers); Assert.assertEquals(env.getParallelism(), ds.count()); Assert.assertEquals(expectedSize, ds.sum(1).collect().get(0).f1.longValue()); }
@Override protected void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); if (!isCollectionExecution()) { Assert.assertTrue(env.getParallelism() > 1); } env.generateSequence(1, 1000) .partitionCustom(new AllZeroPartitioner(), new IdKeySelector<Long>()) .map(new FailExceptInPartitionZeroMapper()) .output(new DiscardingOutputFormat<Long>()); env.execute(); }
@Override protected FlowOptimizer createOptimizer() { FlowOptimizer opt = new FlowOptimizer(); opt.setMaxParallelism(env.getParallelism()); return opt; }
protected StreamPlanEnvironment(ExecutionEnvironment env) { super(); this.env = env; int parallelism = env.getParallelism(); if (parallelism > 0) { setParallelism(parallelism); } else { // determine parallelism setParallelism(GlobalConfiguration.loadConfiguration().getInteger(CoreOptions.DEFAULT_PARALLELISM)); } }
protected StreamPlanEnvironment(ExecutionEnvironment env) { super(); this.env = env; int parallelism = env.getParallelism(); if (parallelism > 0) { setParallelism(parallelism); } else { // determine parallelism setParallelism(GlobalConfiguration.loadConfiguration().getInteger(CoreOptions.DEFAULT_PARALLELISM)); } }
protected StreamPlanEnvironment(ExecutionEnvironment env) { super(); this.env = env; int parallelism = env.getParallelism(); if (parallelism > 0) { setParallelism(parallelism); } else { // determine parallelism setParallelism(GlobalConfiguration.loadConfiguration().getInteger( ConfigConstants.DEFAULT_PARALLELISM_KEY, ConfigConstants.DEFAULT_PARALLELISM)); } }
private void startLocalCluster() { synchronized (lock) { if (localCluster == null) { org.apache.flink.configuration.Configuration configuration = new org.apache.flink.configuration.Configuration(); configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, env.getParallelism() * 2); localCluster = new LocalFlinkMiniCluster(configuration, false); localCluster.start(); } localClusterUsers++; } }
@Override public DataSet translate( FlinkOperator<FlowUnfolder.InputOperator> operator, BatchExecutorContext context) { // get original datasource from operator BoundedDataSource<?> ds = operator.output().getSource().asBounded(); int envParallel = context.getExecutionEnvironment().getParallelism(); DataSourceWrapper<?> wrapper = new DataSourceWrapper<>( ds, splitAssignerFactory, envParallel); return context .getExecutionEnvironment() .createInput(wrapper) .setParallelism(Math.min(envParallel, wrapper.getParallelism())); } }
@Test public void shouldSetParallelismBatch() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setParallelism(42); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(42)); assertThat(bev.getParallelism(), is(42)); }
@Test public void shouldFallbackToDefaultParallelismBatch() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("host:80"); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(1)); assertThat(bev.getParallelism(), is(1)); }
@Test public void shouldFallbackToDefaultParallelismBatch() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("host:80"); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(1)); assertThat(bev.getParallelism(), is(1)); }
@Test public void shouldSetParallelismBatch() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setParallelism(42); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(42)); assertThat(bev.getParallelism(), is(42)); }
@Test public void useDefaultParallelismFromContextBatch() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList()); assertThat(bev, instanceOf(LocalEnvironment.class)); assertThat(options.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); assertThat(bev.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); }
@Test public void shouldInferParallelismFromEnvironmentBatch() throws IOException { String flinkConfDir = extractFlinkConfig(); FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("host:80"); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList(), flinkConfDir); assertThat(options.getParallelism(), is(23)); assertThat(bev.getParallelism(), is(23)); }
@Test public void shouldInferParallelismFromEnvironmentBatch() throws IOException { String flinkConfDir = extractFlinkConfig(); FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("host:80"); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList(), flinkConfDir); assertThat(options.getParallelism(), is(23)); assertThat(bev.getParallelism(), is(23)); }
@Test public void useDefaultParallelismFromContextBatch() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); assertThat(bev.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); }