/** * Creates a {@link LocalStreamEnvironment}. The local execution environment * will run the program in a multi-threaded fashion in the same JVM as the * environment was created in. It will use the parallelism specified in the * parameter. * * @param parallelism * The parallelism for the local environment. * @param configuration * Pass a custom configuration into the cluster * @return A local execution environment with the specified parallelism. */ public static LocalStreamEnvironment createLocalEnvironment(int parallelism, Configuration configuration) { final LocalStreamEnvironment currentEnvironment; currentEnvironment = new LocalStreamEnvironment(configuration); currentEnvironment.setParallelism(parallelism); return currentEnvironment; }
/** * Test test verifies that the execution environment can be used to execute a * single job with multiple slots. */ @Test public void testRunIsolatedJob() throws Exception { LocalStreamEnvironment env = new LocalStreamEnvironment(); assertEquals(1, env.getParallelism()); addSmallBoundedJob(env, 3); env.execute(); }
/** * Test test verifies that the execution environment can be used to execute multiple * bounded streaming jobs after one another. */ @Test public void testMultipleJobsAfterAnother() throws Exception { LocalStreamEnvironment env = new LocalStreamEnvironment(); addSmallBoundedJob(env, 3); env.execute(); addSmallBoundedJob(env, 5); env.execute(); }
/** * Creates a new mini cluster stream environment that configures its local executor with the given configuration. * * @param configuration The configuration used to configure the local executor. */ public LocalStreamEnvironment(@Nonnull Configuration configuration) { if (!ExecutionEnvironment.areExplicitEnvironmentsAllowed()) { throw new InvalidProgramException( "The LocalStreamEnvironment cannot be used when submitting a program through a client, " + "or running in a TestEnvironment context."); } this.configuration = configuration; setParallelism(1); }
/** * Creates a {@link LocalStreamEnvironment}. The local execution environment * will run the program in a multi-threaded fashion in the same JVM as the * environment was created in. The default parallelism of the local * environment is the number of hardware contexts (CPU cores / threads), * unless it was specified differently by {@link PythonStreamExecutionEnvironment#set_parallelism(int)}. * * @param config Pass a custom configuration into the cluster * @return A local execution environment with the specified parallelism. */ public PythonStreamExecutionEnvironment create_local_execution_environment(Configuration config) { return new PythonStreamExecutionEnvironment(new LocalStreamEnvironment(config), new Path(localTmpPath), scriptName); }
public JobExecutionResult execute(String jobName) throws Exception { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName);
public JobExecutionResult execute(String jobName) throws Exception { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); try { exec.start(); return exec.submitJobAndWait(jobGraph, getConfig().isSysoutLoggingEnabled());
@Test public void useDefaultParallelismFromContextStreaming() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); StreamExecutionEnvironment sev = FlinkExecutionEnvironments.createStreamExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); assertThat(sev.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); }
/** * Creates a new mini cluster stream environment that configures its local executor with the given configuration. * * @param configuration The configuration used to configure the local executor. */ public LocalStreamEnvironment(@Nonnull Configuration configuration) { if (!ExecutionEnvironment.areExplicitEnvironmentsAllowed()) { throw new InvalidProgramException( "The LocalStreamEnvironment cannot be used when submitting a program through a client, " + "or running in a TestEnvironment context."); } this.configuration = configuration; setParallelism(1); }
static LocalFlinkMiniCluster execute(LocalStreamEnvironment env, Configuration conf, String jobName) throws Exception { StreamGraph streamGraph = env.getStreamGraph(); streamGraph.setJobName(jobName); JobGraph jobGraph = streamGraph.getJobGraph(); Configuration configuration = new Configuration(conf); configuration.addAll(jobGraph.getJobConfiguration()); configuration.setLong("taskmanager.memory.size", -1L); configuration.setInteger("taskmanager.numberOfTaskSlots", jobGraph.getMaximumParallelism()); LocalFlinkMiniCluster cluster = new LocalFlinkMiniCluster(configuration, true); cluster.start(); cluster.submitJobDetached(jobGraph); return cluster; } }
@Test public void useDefaultParallelismFromContextBatch() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); assertThat(bev.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); }
/** * Creates a {@link LocalStreamEnvironment}. The local execution environment * will run the program in a multi-threaded fashion in the same JVM as the * environment was created in. It will use the parallelism specified in the * parameter. * * @param parallelism * The parallelism for the local environment. * @return A local execution environment with the specified parallelism. */ public static LocalStreamEnvironment createLocalEnvironment(int parallelism) { LocalStreamEnvironment env = new LocalStreamEnvironment(); env.setParallelism(parallelism); return env; }
/** * Creates a new mini cluster stream environment that configures its local executor with the given configuration. * * @param configuration The configuration used to configure the local executor. */ public LocalStreamEnvironment(@Nonnull Configuration configuration) { if (!ExecutionEnvironment.areExplicitEnvironmentsAllowed()) { throw new InvalidProgramException( "The LocalStreamEnvironment cannot be used when submitting a program through a client, " + "or running in a TestEnvironment context."); } this.configuration = configuration; setParallelism(1); }
public JobExecutionResult execute(String jobName) throws Exception { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName);
@Test public void useDefaultParallelismFromContextBatch() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); ExecutionEnvironment bev = FlinkExecutionEnvironments.createBatchExecutionEnvironment( options, Collections.emptyList()); assertThat(bev, instanceOf(LocalEnvironment.class)); assertThat(options.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); assertThat(bev.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); }
/** * Creates a {@link LocalStreamEnvironment}. The local execution environment * will run the program in a multi-threaded fashion in the same JVM as the * environment was created in. It will use the parallelism specified in the * parameter. * * @param parallelism * The parallelism for the local environment. * @param configuration * Pass a custom configuration into the cluster * @return A local execution environment with the specified parallelism. */ public static LocalStreamEnvironment createLocalEnvironment(int parallelism, Configuration configuration) { LocalStreamEnvironment currentEnvironment = new LocalStreamEnvironment(configuration); currentEnvironment.setParallelism(parallelism); return currentEnvironment; }
.createLocalEnvironment().setParallelism(1);
public JobExecutionResult execute(String jobName) throws Exception { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName);
@Test public void useDefaultParallelismFromContextStreaming() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); StreamExecutionEnvironment sev = FlinkExecutionEnvironments.createStreamExecutionEnvironment( options, Collections.emptyList()); assertThat(sev, instanceOf(LocalStreamEnvironment.class)); assertThat(options.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); assertThat(sev.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); }
/** * Creates a {@link LocalStreamEnvironment}. The local execution environment * will run the program in a multi-threaded fashion in the same JVM as the * environment was created in. It will use the parallelism specified in the * parameter. * * @param parallelism * The parallelism for the local environment. * @param configuration * Pass a custom configuration into the cluster * @return A local execution environment with the specified parallelism. */ public static LocalStreamEnvironment createLocalEnvironment(int parallelism, Configuration configuration) { final LocalStreamEnvironment currentEnvironment; currentEnvironment = new LocalStreamEnvironment(configuration); currentEnvironment.setParallelism(parallelism); return currentEnvironment; }