@Override public JobExecutionResult execute(String jobName) throws ProgramInvocationException { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); return executeRemotely(streamGraph, jarFiles); }
/** * Creates a {@link RemoteStreamEnvironment}. The remote environment sends * (parts of) the program to a cluster for execution. Note that all file * paths used in the program must be accessible from the cluster. The * execution will use the specified parallelism. * * @param host * The host name or address of the master (JobManager), where the * program should be executed. * @param port * The port of the master (JobManager), where the program should * be executed. * @param parallelism * The parallelism to use during the execution. * @param jarFiles * The JAR files with code that needs to be shipped to the * cluster. If the program uses user-defined functions, * user-defined input formats, or any libraries, those must be * provided in the JAR files. * @return A remote environment that executes the program on a cluster. */ public static StreamExecutionEnvironment createRemoteEnvironment( String host, int port, int parallelism, String... jarFiles) { RemoteStreamEnvironment env = new RemoteStreamEnvironment(host, port, jarFiles); env.setParallelism(parallelism); return env; }
/** * Executes the remote job. * * <p>Note: This method exposes stream graph internal in the public API, but cannot be removed for backward compatibility. * @param streamGraph * Stream Graph to execute * @param jarFiles * List of jar file URLs to ship to the cluster * @return The result of the job execution, containing elapsed time and accumulators. */ @Deprecated protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL> jarFiles) throws ProgramInvocationException { return executeRemotely(streamGraph, this.getClass().getClassLoader(), getConfig(), jarFiles, host, port, clientConfiguration, globalClasspaths, savepointRestoreSettings); }
/** * Creates a {@link RemoteStreamEnvironment}. The remote environment sends * (parts of) the program to a cluster for execution. Note that all file * paths used in the program must be accessible from the cluster. The * execution will use no parallelism, unless the parallelism is set * explicitly via {@link #setParallelism}. * * @param host * The host name or address of the master (JobManager), where the * program should be executed. * @param port * The port of the master (JobManager), where the program should * be executed. * @param jarFiles * The JAR files with code that needs to be shipped to the * cluster. If the program uses user-defined functions, * user-defined input formats, or any libraries, those must be * provided in the JAR files. * @return A remote environment that executes the program on a cluster. */ public static StreamExecutionEnvironment createRemoteEnvironment( String host, int port, String... jarFiles) { return new RemoteStreamEnvironment(host, port, jarFiles); }
/** * Executes the remote job. * * @param streamGraph * Stream Graph to execute * @param jarFiles * List of jar file URLs to ship to the cluster * @return The result of the job execution, containing elapsed time and accumulators. */ @Override protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL> jarFiles) throws ProgramInvocationException { URL jarUrl; try { jarUrl = flinkILoop.writeFilesToDisk().getAbsoluteFile().toURI().toURL(); } catch (MalformedURLException e) { throw new ProgramInvocationException("Could not write the user code classes to disk.", streamGraph.getJobGraph().getJobID(), e); } List<URL> allJarFiles = new ArrayList<>(jarFiles.size() + 1); allJarFiles.addAll(jarFiles); allJarFiles.add(jarUrl); return super.executeRemotely(streamGraph, allJarFiles); }
@Test public void testRemoteExecutionWithSavepoint() throws Exception { SavepointRestoreSettings restoreSettings = SavepointRestoreSettings.forPath("fakePath"); RemoteStreamEnvironment env = new RemoteStreamEnvironment("fakeHost", 1, null, new String[]{}, null, restoreSettings); env.fromElements(1).map(x -> x * 2); RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class); JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null); PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenReturn(mockedClient); when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.eq(restoreSettings))) .thenReturn(expectedResult); JobExecutionResult actualResult = env.execute("fakeJobName"); Assert.assertEquals(expectedResult, actualResult); } }
String host = super.getHost(); int port = super.getPort(); configuration.addAll(super.getClientConfiguration());
@Override public String toString() { return "Remote Environment (" + this.host + ":" + this.port + " - parallelism = " + (getParallelism() == -1 ? "default" : getParallelism()) + ")"; }
try { client = new StandaloneClusterClient(configuration); client.setPrintStatusDuringExecution(getConfig().isSysoutLoggingEnabled());
/** * Creates a {@link RemoteStreamEnvironment}. The remote environment sends * (parts of) the program to a cluster for execution. Note that all file * paths used in the program must be accessible from the cluster. The * execution will use the specified parallelism. * * @param host * The host name or address of the master (JobManager), where the * program should be executed. * @param port * The port of the master (JobManager), where the program should * be executed. * @param clientConfig * The configuration used by the client that connects to the remote cluster. * @param jarFiles * The JAR files with code that needs to be shipped to the * cluster. If the program uses user-defined functions, * user-defined input formats, or any libraries, those must be * provided in the JAR files. * @return A remote environment that executes the program on a cluster. */ public static StreamExecutionEnvironment createRemoteEnvironment( String host, int port, Configuration clientConfig, String... jarFiles) { return new RemoteStreamEnvironment(host, port, clientConfig, jarFiles); }
StreamGraph streamGraph = streamExecutionEnvironment.getStreamGraph(); streamGraph.setJobName(jobName); return executeRemotely(streamGraph, streamExecutionEnvironment.getClass().getClassLoader(), streamExecutionEnvironment.getConfig(),
@Override public String toString() { return "Remote Environment (" + this.host + ":" + this.port + " - parallelism = " + (getParallelism() == -1 ? "default" : getParallelism()) + ")"; }
client.setPrintStatusDuringExecution(getConfig().isSysoutLoggingEnabled());
/** * Creates a {@link RemoteStreamEnvironment}. The remote environment sends * (parts of) the program to a cluster for execution. Note that all file * paths used in the program must be accessible from the cluster. The * execution will use the specified parallelism. * * @param host * The host name or address of the master (JobManager), where the * program should be executed. * @param port * The port of the master (JobManager), where the program should * be executed. * @param parallelism * The parallelism to use during the execution. * @param jarFiles * The JAR files with code that needs to be shipped to the * cluster. If the program uses user-defined functions, * user-defined input formats, or any libraries, those must be * provided in the JAR files. * @return A remote environment that executes the program on a cluster. */ public static StreamExecutionEnvironment createRemoteEnvironment( String host, int port, int parallelism, String... jarFiles) { RemoteStreamEnvironment env = new RemoteStreamEnvironment(host, port, jarFiles); env.setParallelism(parallelism); return env; }
@Override public JobExecutionResult execute(String jobName) throws ProgramInvocationException { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); return executeRemotely(streamGraph, jarFiles); }
/** * Creates a {@link RemoteStreamEnvironment}. The remote environment sends * (parts of) the program to a cluster for execution. Note that all file * paths used in the program must be accessible from the cluster. The * execution will use no parallelism, unless the parallelism is set * explicitly via {@link #setParallelism}. * * @param host * The host name or address of the master (JobManager), where the * program should be executed. * @param port * The port of the master (JobManager), where the program should * be executed. * @param jarFiles * The JAR files with code that needs to be shipped to the * cluster. If the program uses user-defined functions, * user-defined input formats, or any libraries, those must be * provided in the JAR files. * @return A remote environment that executes the program on a cluster. */ public static StreamExecutionEnvironment createRemoteEnvironment( String host, int port, String... jarFiles) { return new RemoteStreamEnvironment(host, port, jarFiles); }
/** * Executes the remote job. * * @param streamGraph * Stream Graph to execute * @param jarFiles * List of jar file URLs to ship to the cluster * @return The result of the job execution, containing elapsed time and accumulators. */ @Override protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL> jarFiles) throws ProgramInvocationException { URL jarUrl; try { jarUrl = flinkILoop.writeFilesToDisk().getAbsoluteFile().toURI().toURL(); } catch (MalformedURLException e) { throw new ProgramInvocationException("Could not write the user code classes to disk.", e); } List<URL> allJarFiles = new ArrayList<>(jarFiles.size() + 1); allJarFiles.addAll(jarFiles); allJarFiles.add(jarUrl); return super.executeRemotely(streamGraph, allJarFiles); }