/** * Gets the {@link java.lang.ClassLoader} that must be used to load user code classes. * * @return The user code ClassLoader. */ public ClassLoader getUserCodeClassLoader() { if (this.userCodeClassLoader == null) { this.userCodeClassLoader = buildUserCodeClassLoader(jarFiles, classpaths, getClass().getClassLoader()); } return this.userCodeClassLoader; }
public JobWithJars(Plan plan, URL jarFile) throws IOException { this.plan = plan; checkJarFile(jarFile); this.jarFiles = Collections.singletonList(jarFile); this.classpaths = Collections.<URL>emptyList(); }
@Override public JobExecutionResult executePlan(Plan plan) throws Exception { if (plan == null) { throw new IllegalArgumentException("The plan may not be null."); } JobWithJars p = new JobWithJars(plan, this.jarFiles, this.globalClasspaths); return executePlanWithJars(p); }
/** * Runs a program on the Flink cluster to which this client is connected. The call blocks until the * execution is complete, and returns afterwards. * * @param jobWithJars The program to be executed. * @param parallelism The default parallelism to use when running the program. The default parallelism is used * when the program does not set a parallelism by itself. * * @throws CompilerException Thrown, if the compiler encounters an illegal situation. * @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file, * or if the submission failed. That might be either due to an I/O problem, * i.e. the job-manager is unreachable, or due to the fact that the * parallel execution failed. */ public JobSubmissionResult run(JobWithJars jobWithJars, int parallelism, SavepointRestoreSettings savepointSettings) throws CompilerException, ProgramInvocationException { ClassLoader classLoader = jobWithJars.getUserCodeClassLoader(); if (classLoader == null) { throw new IllegalArgumentException("The given JobWithJars does not provide a usercode class loader."); } OptimizedPlan optPlan = getOptimizedPlan(compiler, jobWithJars, parallelism); return run(optPlan, jobWithJars.getJarFiles(), jobWithJars.getClasspaths(), classLoader, savepointSettings); }
uploadedJarUri = new File(uploadedJarLocation).getAbsoluteFile().toURI(); uploadedJarUrl = uploadedJarUri.toURL(); JobWithJars.checkJarFile(uploadedJarUrl); } catch (final IOException e) { throw new RuntimeException("Problem with jar file " + uploadedJarLocation, e); ClassLoader classLoader = JobWithJars.buildUserCodeClassLoader( Collections.<URL>singletonList(uploadedJarUrl), Collections.<URL>emptyList(),
/** * Runs a program on the Flink cluster to which this client is connected. The call blocks until the * execution is complete, and returns afterwards. * * @param jobWithJars The program to be executed. * @param parallelism The default parallelism to use when running the program. The default parallelism is used * when the program does not set a parallelism by itself. * * @throws CompilerException Thrown, if the compiler encounters an illegal situation. * @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file, * or if the submission failed. That might be either due to an I/O problem, * i.e. the job-manager is unreachable, or due to the fact that the * parallel execution failed. */ public JobSubmissionResult run(JobWithJars jobWithJars, int parallelism, SavepointRestoreSettings savepointSettings, boolean detached) throws CompilerException, ProgramInvocationException { ClassLoader classLoader = jobWithJars.getUserCodeClassLoader(); if (classLoader == null) { throw new IllegalArgumentException("The given JobWithJars does not provide a usercode class loader."); } OptimizedPlan optPlan = getOptimizedPlan(compiler, jobWithJars, parallelism); return run(optPlan, jobWithJars.getJarFiles(), jobWithJars.getClasspaths(), jobWithJars.getLibjars(), jobWithJars.getFiles(), classLoader, savepointSettings, detached); }
/** * Creates the optimized plan for a given program, using this client's compiler. * * @param prog The program to be compiled. * @return The compiled and optimized plan, as returned by the compiler. * @throws CompilerException Thrown, if the compiler encounters an illegal situation. * @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file. */ private static OptimizedPlan getOptimizedPlan(Optimizer compiler, JobWithJars prog, int parallelism) throws CompilerException, ProgramInvocationException { return getOptimizedPlan(compiler, prog.getPlan(), parallelism); }
/** * Runs a program on the Flink cluster to which this client is connected. The call blocks until the * execution is complete, and returns afterwards. * * @param jobWithJars The program to be executed. * @param parallelism The default parallelism to use when running the program. The default parallelism is used * when the program does not set a parallelism by itself. * * @throws CompilerException Thrown, if the compiler encounters an illegal situation. * @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file, * or if the submission failed. That might be either due to an I/O problem, * i.e. the job-manager is unreachable, or due to the fact that the * parallel execution failed. */ public JobSubmissionResult run(JobWithJars jobWithJars, int parallelism, SavepointRestoreSettings savepointSettings) throws CompilerException, ProgramInvocationException { ClassLoader classLoader = jobWithJars.getUserCodeClassLoader(); if (classLoader == null) { throw new IllegalArgumentException("The given JobWithJars does not provide a usercode class loader."); } OptimizedPlan optPlan = getOptimizedPlan(compiler, jobWithJars, parallelism); return run(optPlan, jobWithJars.getJarFiles(), jobWithJars.getClasspaths(), classLoader, savepointSettings); }
this.userCodeClassLoader = JobWithJars.buildUserCodeClassLoader(getAllLibraries(), classpaths, getClass().getClassLoader());
public JobWithJars(Plan plan, List<URL> jarFiles, List<URL> classpaths) throws IOException { this.plan = plan; this.jarFiles = new ArrayList<URL>(jarFiles.size()); this.classpaths = new ArrayList<URL>(classpaths.size()); for (URL jarFile: jarFiles) { checkJarFile(jarFile); this.jarFiles.add(jarFile); } for (URL path: classpaths) { this.classpaths.add(path); } }
/** * Returns the plan without the required jars when the files are already provided by the cluster. * * @return The plan without attached jar files. * @throws ProgramInvocationException */ public JobWithJars getPlanWithoutJars() throws ProgramInvocationException { if (isUsingProgramEntryPoint()) { return new JobWithJars(getPlan(), Collections.<URL>emptyList(), classpaths, userCodeClassLoader); } else { throw new ProgramInvocationException("Cannot create a " + JobWithJars.class.getSimpleName() + " for a program that is using the interactive mode.", getPlan().getJobId()); } }
/** * Runs a program on the Flink cluster to which this client is connected. The call blocks until the * execution is complete, and returns afterwards. * * @param jobWithJars The program to be executed. * @param parallelism The default parallelism to use when running the program. The default parallelism is used * when the program does not set a parallelism by itself. * * @throws CompilerException Thrown, if the compiler encounters an illegal situation. * @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file, * or if the submission failed. That might be either due to an I/O problem, * i.e. the job-manager is unreachable, or due to the fact that the * parallel execution failed. */ public JobSubmissionResult run(JobWithJars jobWithJars, int parallelism, SavepointRestoreSettings savepointSettings) throws CompilerException, ProgramInvocationException { ClassLoader classLoader = jobWithJars.getUserCodeClassLoader(); if (classLoader == null) { throw new IllegalArgumentException("The given JobWithJars does not provide a usercode class loader."); } OptimizedPlan optPlan = getOptimizedPlan(compiler, jobWithJars, parallelism); return run(optPlan, jobWithJars.getJarFiles(), jobWithJars.getClasspaths(), classLoader, savepointSettings); }
/** * Creates the optimized plan for a given program, using this client's compiler. * * @param prog The program to be compiled. * @return The compiled and optimized plan, as returned by the compiler. * @throws CompilerException Thrown, if the compiler encounters an illegal situation. * @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file. */ private static OptimizedPlan getOptimizedPlan(Optimizer compiler, JobWithJars prog, int parallelism) throws CompilerException, ProgramInvocationException { return getOptimizedPlan(compiler, prog.getPlan(), parallelism); }
ClassLoader userCodeClassLoader = JobWithJars.buildUserCodeClassLoader(jarFiles, globalClasspaths, envClassLoader);
private static void checkJarFile(URL jarfile) throws ProgramInvocationException { try { JobWithJars.checkJarFile(jarfile); } catch (IOException e) { throw new ProgramInvocationException(e.getMessage()); } catch (Throwable t) { throw new ProgramInvocationException("Cannot access jar file" + (t.getMessage() == null ? "." : ": " + t.getMessage()), t); } }
@Override public JobExecutionResult execute(String jobName) throws Exception { Plan p = createProgramPlan(jobName); JobWithJars toRun = new JobWithJars(p, this.jarFilesToAttach, this.classpathsToAttach, this.userCodeClassLoader); this.lastJobExecutionResult = client.run(toRun, getParallelism(), savepointSettings).getJobExecutionResult(); return this.lastJobExecutionResult; }
/** * Creates the optimized plan for a given program, using this client's compiler. * * @param prog The program to be compiled. * @return The compiled and optimized plan, as returned by the compiler. * @throws CompilerException Thrown, if the compiler encounters an illegal situation. * @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file. */ private static OptimizedPlan getOptimizedPlan(Optimizer compiler, JobWithJars prog, int parallelism) throws CompilerException, ProgramInvocationException { return getOptimizedPlan(compiler, prog.getPlan(), parallelism); }
/** * Gets the {@link java.lang.ClassLoader} that must be used to load user code classes. * * @return The user code ClassLoader. */ public ClassLoader getUserCodeClassLoader() { if (this.userCodeClassLoader == null) { this.userCodeClassLoader = buildUserCodeClassLoader(jarFiles, classpaths, getClass().getClassLoader()); } return this.userCodeClassLoader; }
JobWithJars.checkJarFile(url); dependencies.add(url); JobWithJars.checkJarFile(url); dependencies.add(url);