client.shutdown(); } catch (Exception e) { LOG.info("Could not properly shut down the client.", e); client.setPrintStatusDuringExecution(runOptions.getStdoutLogging()); client.setDetached(runOptions.getDetachedMode()); LOG.debug("Client slots is set to {}", client.getMaxSlots()); if (client.getMaxSlots() != MAX_SLOTS_UNKNOWN && userParallelism == -1) { logAndSysout("Using the parallelism provided by the remote cluster (" + client.getMaxSlots() + "). " + "To use another parallelism, set it at the ./bin/flink client."); userParallelism = client.getMaxSlots(); } else if (ExecutionConfig.PARALLELISM_DEFAULT == userParallelism) { userParallelism = defaultParallelism; if (clusterId == null && !client.isDetached()) { client.shutDownCluster(); } catch (final Exception e) { LOG.info("Could not properly terminate the Flink cluster.", e); client.shutdown(); } catch (Exception e) { LOG.info("Could not properly shut down the client.", e);
@Override public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { if (isDetached()) { return super.runDetached(jobGraph, classLoader); } else { return super.run(jobGraph, classLoader); } } }
@Override public void run() { try { client.setDetached(false); client.submitJob(jobGraph, KafkaConsumerTestBase.class.getClassLoader()); } catch (Throwable t) { jobError.set(t); } } };
public JobSubmissionResult run(FlinkPlan compiledPlan, List<URL> libraries, List<URL> classpaths, ClassLoader classLoader, SavepointRestoreSettings savepointSettings) throws ProgramInvocationException { JobGraph job = getJobGraph(flinkConfig, compiledPlan, libraries, classpaths, savepointSettings); return submitJob(job, classLoader); }
client.setPrintStatusDuringExecution(executionConfig.isSysoutLoggingEnabled()); return client.run(streamGraph, jarFiles, globalClasspaths, userCodeClassLoader, savepointRestoreSettings) .getJobExecutionResult(); client.shutdown(); } catch (Exception e) { LOG.warn("Could not properly shut down the cluster client.", e);
protected void runAndCancelJob(Plan plan, final int msecsTillCanceling, int maxTimeTillCanceled) throws Exception { // submit job final JobGraph jobGraph = getJobGraph(plan); ClusterClient<?> client = CLUSTER.getClusterClient(); client.setDetached(true); JobSubmissionResult jobSubmissionResult = client.submitJob(jobGraph, CancelingTestBase.class.getClassLoader()); Deadline submissionDeadLine = new FiniteDuration(2, TimeUnit.MINUTES).fromNow(); JobStatus jobStatus = client.getJobStatus(jobSubmissionResult.getJobID()).get(GET_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); while (jobStatus != JobStatus.RUNNING && submissionDeadLine.hasTimeLeft()) { Thread.sleep(50); jobStatus = client.getJobStatus(jobSubmissionResult.getJobID()).get(GET_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); } if (jobStatus != JobStatus.RUNNING) { Assert.fail("Job not in state RUNNING."); } Thread.sleep(msecsTillCanceling); client.cancel(jobSubmissionResult.getJobID()); Deadline cancelDeadline = new FiniteDuration(maxTimeTillCanceled, TimeUnit.MILLISECONDS).fromNow(); JobStatus jobStatusAfterCancel = client.getJobStatus(jobSubmissionResult.getJobID()).get(GET_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); while (jobStatusAfterCancel != JobStatus.CANCELED && cancelDeadline.hasTimeLeft()) { Thread.sleep(50); jobStatusAfterCancel = client.getJobStatus(jobSubmissionResult.getJobID()).get(GET_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); } if (jobStatusAfterCancel != JobStatus.CANCELED) { Assert.fail("Failed to cancel job with ID " + jobSubmissionResult.getJobID() + '.'); } }
client.setDetached(true); client.submitJob(jobGraph, RescalingITCase.class.getClassLoader()); CompletableFuture<String> savepointPathFuture = client.triggerSavepoint(jobID, null); client.cancel(jobID); client.setDetached(false); client.submitJob(scaledJobGraph, RescalingITCase.class.getClassLoader()); } catch (JobExecutionException exception) { if (exception.getCause() instanceof IllegalStateException) {
yarnApplicationId = clusterClient.getClusterId(); final LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); System.out.println("JobManager Web Interface: " + clusterClient.getWebInterfaceURL()); } catch (Exception e) { try { clusterClient.shutdown(); } catch (Exception ex) { LOG.info("Could not properly shutdown cluster client.", ex);
webInterfaceUrl = clusterClient.getWebInterfaceURL(); } catch (Exception e) { webInterfaceUrl = "N/A"; result.setClusterInformation(clusterClient.getClusterId(), webInterfaceUrl); clusterClient.setDetached(false); final JobExecutionResult jobResult = clusterClient .submitJob(jobGraph, context.getClassLoader()) clusterClient.setDetached(true); clusterClient.submitJob(jobGraph, context.getClassLoader()); clusterClient.shutdown();
@Override public void stop() throws Exception { synchronized (lock) { if (client != null) { client.shutdown(); client = null; } } }
if (hasUserJarsInClassPath(prog.getAllLibraries())) { jobWithJars = prog.getPlanWithoutJars(); } else { return run(jobWithJars, parallelism, prog.getSavepointSettings()); log.info("Starting program in interactive mode (detached: {})", isDetached()); if (hasUserJarsInClassPath(prog.getAllLibraries())) { libraries = Collections.emptyList(); } else { prog.getClasspaths(), prog.getUserCodeClassLoader(), parallelism, isDetached(), prog.getSavepointSettings()); ContextEnvironment.setAsContext(factory); throw new ProgramMissingJobException("The program didn't contain a Flink job."); if (isDetached()) {
private <T> void deployJobOnNewCluster( ClusterDescriptor<T> clusterDescriptor, JobGraph jobGraph, Result<T> result, ClassLoader classLoader) throws Exception { ClusterClient<T> clusterClient = null; try { // deploy job cluster with job attached clusterClient = clusterDescriptor.deployJobCluster(context.getClusterSpec(), jobGraph, false); // save information about the new cluster result.setClusterInformation(clusterClient.getClusterId(), clusterClient.getWebInterfaceURL()); // get result if (awaitJobResult) { // we need to hard cast for now final JobExecutionResult jobResult = ((RestClusterClient<T>) clusterClient) .requestJobResult(jobGraph.getJobID()) .get() .toJobExecutionResult(context.getClassLoader()); // throws exception if job fails executionResultBucket.add(jobResult); } } finally { try { if (clusterClient != null) { clusterClient.shutdown(); } } catch (Exception e) { // ignore } } }
/** * Finishes this Context Environment's execution by explicitly running the plan constructed. */ JobSubmissionResult finalizeExecute() throws ProgramInvocationException { return client.run(detachedPlan, jarFilesToAttach, classpathsToAttach, userCodeClassLoader, savepointSettings); }
client.setPrintStatusDuringExecution(options.getStdoutLogging()); client.setDetached(options.getDetachedMode()); LOG.debug("Client slots is set to {}", client.getMaxSlots()); if (client.getMaxSlots() != -1 && userParallelism == -1) { logAndSysout("Using the parallelism provided by the remote cluster (" + client.getMaxSlots()+"). " + "To use another parallelism, set it at the ./bin/flink client."); userParallelism = client.getMaxSlots(); if (client != null) { try { client.shutdown(); } catch (Exception e) { LOG.warn("Could not properly shut down the cluster client.", e);
public static void main(String[] args) throws Exception { LauncherOptions launcherOptions = new LauncherOptionParser(args).getLauncherOptions(); String mode = launcherOptions.getMode(); List<String> argList = initFlinkxArgList(launcherOptions); if(mode.equals(ClusterMode.local.name())) { String[] localArgs = argList.toArray(new String[argList.size()]); com.dtstack.flinkx.Main.main(localArgs); } else { ClusterClient clusterClient = ClusterClientFactory.createClusterClient(launcherOptions); String monitor = clusterClient.getWebInterfaceURL(); argList.add("-monitor"); argList.add(monitor); String pluginRoot = launcherOptions.getPlugin(); String content = launcherOptions.getJob(); File jarFile = new File(pluginRoot + File.separator + "flinkx.jar"); List<URL> urlList = analyzeUserClasspath(content, pluginRoot); String[] remoteArgs = argList.toArray(new String[argList.size()]); PackagedProgram program = new PackagedProgram(jarFile, urlList, remoteArgs); clusterClient.run(program, launcherOptions.getParallelism()); clusterClient.shutdown(); } } }
private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception { final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000); final JobID jobId = jobGraph.getJobID(); StatefulCounter.resetForTest(parallelism); MiniClusterWithClientResource cluster = clusterFactory.get(); cluster.before(); ClusterClient<?> client = cluster.getClusterClient(); try { client.setDetached(true); client.submitJob(jobGraph, SavepointITCase.class.getClassLoader()); StatefulCounter.getProgressLatch().await(); return client.triggerSavepoint(jobId, null).get(); } finally { cluster.after(); StatefulCounter.resetForTest(parallelism); } }
/** * Tests that a CLI call with a custom savepoint directory target is * forwarded correctly to the cluster client. */ @Test public void testTriggerSavepointCustomTarget() throws Exception { replaceStdOutAndStdErr(); JobID jobId = new JobID(); String savepointDirectory = "customTargetDirectory"; final ClusterClient<String> clusterClient = createClusterClient(savepointDirectory); try { MockedCliFrontend frontend = new MockedCliFrontend(clusterClient); String[] parameters = { jobId.toString(), savepointDirectory }; frontend.savepoint(parameters); verify(clusterClient, times(1)) .triggerSavepoint(eq(jobId), eq(savepointDirectory)); assertTrue(buffer.toString().contains(savepointDirectory)); } finally { clusterClient.shutdown(); restoreStdOutAndStdErr(); } }
private void testProgram( final MiniClusterWithClientResource cluster, final int dataVolumeGb, final boolean useForwarder, final boolean isSlowSender, final boolean isSlowReceiver, final int parallelism) throws Exception { ClusterClient<?> client = cluster.getClusterClient(); client.setDetached(false); client.setPrintStatusDuringExecution(false); JobExecutionResult jer = (JobExecutionResult) client.submitJob( createJobGraph( dataVolumeGb, useForwarder, isSlowSender, isSlowReceiver, parallelism), getClass().getClassLoader()); long dataVolumeMbit = dataVolumeGb * 8192; long runtimeSecs = jer.getNetRuntime(TimeUnit.SECONDS); int mbitPerSecond = (int) (((double) dataVolumeMbit) / runtimeSecs); LOG.info(String.format("Test finished with throughput of %d MBit/s (runtime [secs]: %d, " + "data volume [gb/mbits]: %d/%d)", mbitPerSecond, runtimeSecs, dataVolumeGb, dataVolumeMbit)); }
@VisibleForTesting void start(AthenaXYarnClusterDescriptor descriptor, JobGraph job) throws Exception { ClusterClient<ApplicationId> client = descriptor.deploy(); try { client.runDetached(job, null); stopAfterJob(client, job.getJobID()); } finally { client.shutdown(); } }