@Test public void testExternalProgram() throws Exception { TestEnvironment.setAsContext( MINI_CLUSTER, PARALLELISM, Collections.singleton(new Path(JAR_FILE)), Collections.emptyList()); String testData = getClass().getResource(TEST_DATA_FILE).toString(); PackagedProgram program = new PackagedProgram(new File(JAR_FILE), new String[]{testData}); program.invokeInteractiveModeForExecution(); } }
program = new PackagedProgram(f, clazz, new String[0]); } catch (Exception ignored) { JarListInfo.JarEntryInfo jarEntryInfo = new JarListInfo.JarEntryInfo(clazz, program.getDescription()); jarEntryList.add(jarEntryInfo);
Thread.currentThread().setContextClassLoader(prog.getUserCodeClassLoader()); if (prog.isUsingProgramEntryPoint()) { if (hasUserJarsInClassPath(prog.getAllLibraries())) { jobWithJars = prog.getPlanWithoutJars(); } else { jobWithJars = prog.getPlanWithJars(); return run(jobWithJars, parallelism, prog.getSavepointSettings()); else if (prog.isUsingInteractiveMode()) { log.info("Starting program in interactive mode (detached: {})", isDetached()); if (hasUserJarsInClassPath(prog.getAllLibraries())) { libraries = Collections.emptyList(); } else { libraries = prog.getAllLibraries(); prog.getClasspaths(), prog.getUserCodeClassLoader(), parallelism, isDetached(), prog.getSavepointSettings()); ContextEnvironment.setAsContext(factory); prog.invokeInteractiveModeForExecution(); if (lastJobExecutionResult == null && factory.getLastEnvCreated() == null) { throw new ProgramMissingJobException("The program didn't contain a Flink job.");
public static JobGraph getJobGraph(Configuration flinkConfig, PackagedProgram prog, FlinkPlan optPlan, SavepointRestoreSettings savepointSettings) throws ProgramInvocationException { return getJobGraph(flinkConfig, optPlan, prog.getAllLibraries(), prog.getClasspaths(), savepointSettings); }
/** * Returns the plan with all required jars. * * @return The plan with attached jar files. * @throws ProgramInvocationException */ public JobWithJars getPlanWithJars() throws ProgramInvocationException { if (isUsingProgramEntryPoint()) { return new JobWithJars(getPlan(), getAllLibraries(), classpaths, userCodeClassLoader); } else { throw new ProgramInvocationException("Cannot create a " + JobWithJars.class.getSimpleName() + " for a program that is using the interactive mode.", getPlan().getJobId()); } }
public static FlinkPlan getOptimizedPlan(Optimizer compiler, PackagedProgram prog, int parallelism) throws CompilerException, ProgramInvocationException { Thread.currentThread().setContextClassLoader(prog.getUserCodeClassLoader()); if (prog.isUsingProgramEntryPoint()) { return getOptimizedPlan(compiler, prog.getPlanWithJars(), parallelism); } else if (prog.isUsingInteractiveMode()) { // temporary hack to support the optimizer plan preview OptimizerPlanEnvironment env = new OptimizerPlanEnvironment(compiler); if (parallelism > 0) { env.setParallelism(parallelism); } return env.getOptimizedPlan(prog); } else { throw new RuntimeException("Couldn't determine program mode."); } }
Thread.currentThread().setContextClassLoader(this.getUserCodeClassLoader()); List<DataSinkNode> previewPlan; if (isUsingProgramEntryPoint()) { previewPlan = Optimizer.createPreOptimizedPlan(getPlan()); else if (isUsingInteractiveMode()) { invokeInteractiveModeForExecution(); return env.preview; } else { throw new ProgramInvocationException("The program caused an error: ", getPlan().getJobId(), t); throw new ProgramInvocationException( "The program plan could not be fetched. The program silently swallowed the control flow exceptions.", getPlan().getJobId());
private PackagedProgram createPackagedProgram() throws FlinkException { try { final Class<?> mainClass = getClass().getClassLoader().loadClass(jobClassName); return new PackagedProgram(mainClass, programArguments); } catch (ClassNotFoundException | ProgramInvocationException e) { throw new FlinkException("Could not load the provided entrypoint class.", e); } } }
/** * Creates a Packaged program from the given command line options. * * @return A PackagedProgram (upon success) */ PackagedProgram buildProgram(ProgramOptions options) throws FileNotFoundException, ProgramInvocationException { String[] programArgs = options.getProgramArgs(); String jarFilePath = options.getJarFilePath(); List<URL> classpaths = options.getClasspaths(); if (jarFilePath == null) { throw new IllegalArgumentException("The program JAR file was not specified."); } File jarFile = new File(jarFilePath); // Check if JAR file exists if (!jarFile.exists()) { throw new FileNotFoundException("JAR file does not exist: " + jarFile); } else if (!jarFile.isFile()) { throw new FileNotFoundException("JAR file is not a file: " + jarFile); } // Get assembler class String entryPointClass = options.getEntryPointClassName(); PackagedProgram program = entryPointClass == null ? new PackagedProgram(jarFile, classpaths, programArgs) : new PackagedProgram(jarFile, classpaths, entryPointClass, programArgs); program.setSavepointRestoreSettings(options.getSavepointRestoreSettings()); return program; }
checkJarFile(jarFileUrl); entryPointClassName = getEntryPointClassNameFromJar(jarFileUrl); this.extractedTempLibraries = extractContainedLibraries(jarFileUrl); this.classpaths = classpaths; this.userCodeClassLoader = JobWithJars.buildUserCodeClassLoader(getAllLibraries(), classpaths, getClass().getClassLoader()); this.mainClass = loadMainClass(entryPointClassName, userCodeClassLoader); if (!hasMainMethod(mainClass)) { throw new ProgramInvocationException("The given program class implements the " + Program.class.getName() + " interface, but cannot be instantiated. " + } else if (hasMainMethod(mainClass)) { this.program = null; } else {
int defaultParallelism, JobID jobID) throws ProgramInvocationException { Thread.currentThread().setContextClassLoader(packagedProgram.getUserCodeClassLoader()); final Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration); final FlinkPlan flinkPlan; if (packagedProgram.isUsingProgramEntryPoint()) { final JobWithJars jobWithJars = packagedProgram.getPlanWithJars(); } else if (packagedProgram.isUsingInteractiveMode()) { final OptimizerPlanEnvironment optimizerPlanEnvironment = new OptimizerPlanEnvironment(optimizer); jobGraph.setSavepointRestoreSettings(packagedProgram.getSavepointSettings()); } else { final JobGraphGenerator jobGraphGenerator = new JobGraphGenerator(configuration); for (URL url : packagedProgram.getAllLibraries()) { try { jobGraph.addJar(new Path(url.toURI())); jobGraph.setClasspaths(packagedProgram.getClasspaths());
@Test public void testGetPreviewPlan() { try { PackagedProgram prog = new PackagedProgram(new File(CliFrontendTestUtils.getTestJarPath())); final PrintStream out = System.out; final PrintStream err = System.err; try { System.setOut(new PrintStream(new NullOutputStream())); System.setErr(new PrintStream(new NullOutputStream())); Assert.assertNotNull(prog.getPreviewPlan()); } finally { System.setOut(out); System.setErr(err); } } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail("Test is erroneous: " + e.getMessage()); } }
protected Tuple2<JobGraph, ClassLoader> getJobGraphAndClassLoader(JarActionHandlerConfig config) throws Exception { // generate the graph JobGraph graph = null; PackagedProgram program = new PackagedProgram( new File(jarDir, config.getJarFile()), config.getEntryClass(), config.getProgramArgs()); ClassLoader classLoader = program.getUserCodeClassLoader(); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); FlinkPlan plan = ClusterClient.getOptimizedPlan(optimizer, program, config.getParallelism()); if (plan instanceof StreamingPlan) { graph = ((StreamingPlan) plan).getJobGraph(); } else if (plan instanceof OptimizedPlan) { graph = new JobGraphGenerator().compileJobGraph((OptimizedPlan) plan); } if (graph == null) { throw new CompilerException("A valid job graph couldn't be generated for the jar."); } // Set the savepoint settings graph.setSavepointRestoreSettings(config.getSavepointRestoreSettings()); for (URL jar : program.getAllLibraries()) { try { graph.addJar(new Path(jar.toURI())); } catch (URISyntaxException e) { throw new ProgramInvocationException("Invalid jar path. Unexpected error. :("); } } return Tuple2.of(graph, classLoader); }
String description = program.getDescription(); if (description != null) { System.out.println(); program.deleteExtractedLibraries();
/** * Deletes all temporary files created for contained packaged libraries. */ public void deleteExtractedLibraries() { deleteExtractedLibraries(this.extractedTempLibraries); this.extractedTempLibraries.clear(); }
when(packagedProgramMock.isUsingInteractiveMode()).thenReturn(true); doAnswer(new Answer<Void>() { @Override return null; }).when(packagedProgramMock).invokeInteractiveModeForExecution();
/** * This test verifies correct job submission messaging logic and plan translation calls. */ @Test public void shouldSubmitToJobClient() throws Exception { jobManagerSystem.actorOf( Props.create(SuccessReturningActor.class), JobMaster.JOB_MANAGER_NAME); StandaloneClusterClient out = new StandaloneClusterClient(config); out.setDetached(true); JobSubmissionResult result = out.run(program.getPlanWithJars(), 1); assertNotNull(result); program.deleteExtractedLibraries(); }
public static JobGraph getJobGraph(Configuration flinkConfig, PackagedProgram prog, FlinkPlan optPlan, SavepointRestoreSettings savepointSettings) throws ProgramInvocationException { return getJobGraph(flinkConfig, optPlan, prog.getAllLibraries(), prog.getClasspaths(), prog.getLibjars(), prog.getFiles(), savepointSettings); }
ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) { @Override public Class<?> loadClass(String name) throws ClassNotFoundException { when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader); assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName()); assertArrayEquals(reducedArguments, prog.getArguments());