@Override public JobExecutionResult execute(String jobName) throws Exception { Plan p = createProgramPlan(jobName); JobWithJars toRun = new JobWithJars(p, this.jarFilesToAttach, this.classpathsToAttach, this.userCodeClassLoader); this.lastJobExecutionResult = client.run(toRun, getParallelism(), savepointSettings).getJobExecutionResult(); return this.lastJobExecutionResult; }
@Override public ExecutionEnvironment createExecutionEnvironment() { if (isDetached && lastEnvCreated != null) { throw new InvalidProgramException("Multiple environments cannot be created in detached mode"); } lastEnvCreated = isDetached ? new DetachedEnvironment(client, jarFilesToAttach, classpathsToAttach, userCodeClassLoader, savepointSettings) : new ContextEnvironment(client, jarFilesToAttach, classpathsToAttach, userCodeClassLoader, savepointSettings); if (defaultParallelism > 0) { lastEnvCreated.setParallelism(defaultParallelism); } return lastEnvCreated; }
@Override public JobExecutionResult execute(String jobName) throws Exception { Preconditions.checkNotNull(jobName, "Streaming Job name should not be null."); StreamGraph streamGraph = this.getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); // execute the programs if (ctx instanceof DetachedEnvironment) { LOG.warn("Job was executed in detached mode, the results will be available on completion."); ((DetachedEnvironment) ctx).setDetachedPlan(streamGraph); return DetachedEnvironment.DetachedJobExecutionResult.INSTANCE; } else { return ctx .getClient() .run(streamGraph, ctx.getJars(), ctx.getClasspaths(), ctx.getUserCodeClassLoader(), ctx.getSavepointRestoreSettings()) .getJobExecutionResult(); } } }
@Override public String toString() { return "Context Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism()) + ") : " + getIdString(); }
protected StreamContextEnvironment(ContextEnvironment ctx) { this.ctx = ctx; if (ctx.getParallelism() > 0) { setParallelism(ctx.getParallelism()); } }
ContextEnvironment ce = new ContextEnvironment(); ce.setName(getName()); ce.setOverride(false); ce.setType("java.lang.String"); String value = StringUtils.remove(context.getServletContext() .getContextPath(), '/'); ce.setValue(value); logger.debug(String.format("Adding env entry '%s' with value '%s'", getName(), value));
static void setAsContext(ContextEnvironmentFactory factory) { initializeContextEnvironment(factory); }
/** * This part is just to add classpath for the jar when reading remote execution, and will not submit jar from a local * @param env * @param classPathSet * @throws NoSuchFieldException * @throws IllegalAccessException */ private static void addEnvClassPath(StreamExecutionEnvironment env, Set<URL> classPathSet) throws NoSuchFieldException, IllegalAccessException { if(env instanceof StreamContextEnvironment){ Field field = env.getClass().getDeclaredField("ctx"); field.setAccessible(true); ContextEnvironment contextEnvironment= (ContextEnvironment) field.get(env); for(URL url : classPathSet){ contextEnvironment.getClasspaths().add(url); } } }
try { for (final URL url : ((ContextEnvironment) ExecutionEnvironment.getExecutionEnvironment()) .getJars()) {
client = ((ContextEnvironment) env).getClient();
@Override public String toString() { return "Context Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism()) + ") : " + getIdString(); }
protected StreamContextEnvironment(ContextEnvironment ctx) { this.ctx = ctx; if (ctx.getParallelism() > 0) { setParallelism(ctx.getParallelism()); } }
static void setAsContext(ContextEnvironmentFactory factory) { initializeContextEnvironment(factory); }
@Override public JobExecutionResult execute(String jobName) throws Exception { Preconditions.checkNotNull(jobName, "Streaming Job name should not be null."); StreamGraph streamGraph = this.getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); // execute the programs if (ctx instanceof DetachedEnvironment) { LOG.warn("Job was executed in detached mode, the results will be available on completion."); ((DetachedEnvironment) ctx).setDetachedPlan(streamGraph); return DetachedEnvironment.DetachedJobExecutionResult.INSTANCE; } else { return ctx .getClient() .run(streamGraph, ctx.getJars(), ctx.getClasspaths(), ctx.getUserCodeClassLoader(), ctx.getSavepointRestoreSettings()) .getJobExecutionResult(); } } }
@Override public String getExecutionPlan() throws Exception { Plan plan = createProgramPlan("unnamed job"); OptimizedPlan op = ClusterClient.getOptimizedPlan(client.compiler, plan, getParallelism()); PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator(); return gen.getOptimizerPlanAsJSON(op); }
@Override public String toString() { return "Context Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism()) + ") : " + getIdString(); }
protected StreamContextEnvironment(ContextEnvironment ctx) { this.ctx = ctx; if (ctx.getParallelism() > 0) { setParallelism(ctx.getParallelism()); } }
@Override public ExecutionEnvironment createExecutionEnvironment() { if (isDetached && lastEnvCreated != null) { throw new InvalidProgramException("Multiple environments cannot be created in detached mode"); } lastEnvCreated = isDetached ? new DetachedEnvironment(client, jarFilesToAttach, classpathsToAttach, userCodeClassLoader, savepointSettings) : new ContextEnvironment(client, jarFilesToAttach, classpathsToAttach, userCodeClassLoader, savepointSettings); if (defaultParallelism > 0) { lastEnvCreated.setParallelism(defaultParallelism); } return lastEnvCreated; }
static void setAsContext(ContextEnvironmentFactory factory) { initializeContextEnvironment(factory); }
@Override public JobExecutionResult execute(String jobName) throws Exception { Preconditions.checkNotNull("Streaming Job name should not be null."); StreamGraph streamGraph = this.getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); // execute the programs if (ctx instanceof DetachedEnvironment) { LOG.warn("Job was executed in detached mode, the results will be available on completion."); ((DetachedEnvironment) ctx).setDetachedPlan(streamGraph); return DetachedEnvironment.DetachedJobExecutionResult.INSTANCE; } else { return ctx .getClient() .run(streamGraph, ctx.getJars(), ctx.getClasspaths(), ctx.getUserCodeClassLoader(), ctx.getSavepointRestoreSettings()) .getJobExecutionResult(); } } }