/** * Reattaches to a running from the supplied job id. * @param jobID The job id of the job to attach to * @return The JobExecutionResult for the jobID * @throws JobExecutionException if an error occurs during monitoring the job execution */ public JobExecutionResult retrieveJob(JobID jobID) throws JobExecutionException { final ActorSystem actorSystem; try { actorSystem = actorSystemLoader.get(); } catch (FlinkException fe) { throw new JobExecutionException( jobID, "Could not start the ActorSystem needed to talk to the JobManager.", fe); } final JobListeningContext listeningContext = JobClient.attachToRunningJob( jobID, flinkConfig, actorSystem, highAvailabilityServices, timeout, printStatusDuringExecution); return JobClient.awaitJobResult(listeningContext); }
/** * Reattaches to a running job with the given job id. * * @param jobID The job id of the job to attach to * @return The JobExecutionResult for the jobID * @throws JobExecutionException if an error occurs during monitoring the job execution */ public JobListeningContext connectToJob(JobID jobID) throws JobExecutionException { final ActorSystem actorSystem; try { actorSystem = actorSystemLoader.get(); } catch (FlinkException fe) { throw new JobExecutionException( jobID, "Could not start the ActorSystem needed to talk to the JobManager.", fe); } return JobClient.attachToRunningJob( jobID, flinkConfig, actorSystem, highAvailabilityServices, timeout, printStatusDuringExecution); }
private static void printJobStatusMessages(List<JobStatusMessage> jobs) { SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss"); Comparator<JobStatusMessage> startTimeComparator = (o1, o2) -> (int) (o1.getStartTime() - o2.getStartTime()); Comparator<Map.Entry<JobStatus, List<JobStatusMessage>>> statusComparator = (o1, o2) -> String.CASE_INSENSITIVE_ORDER.compare(o1.getKey().toString(), o2.getKey().toString()); Map<JobStatus, List<JobStatusMessage>> jobsByState = jobs.stream().collect(Collectors.groupingBy(JobStatusMessage::getJobState)); jobsByState.entrySet().stream() .sorted(statusComparator) .map(Map.Entry::getValue).flatMap(List::stream).sorted(startTimeComparator) .forEachOrdered(job -> System.out.println(dateFormat.format(new Date(job.getStartTime())) + " : " + job.getJobId() + " : " + job.getJobName() + " (" + job.getJobState() + ")")); }
private static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); }
@Override public CompletableFuture<Collection<JobStatusMessage>> listJobs() { return sendRequest(JobsOverviewHeaders.getInstance()) .thenApply( (multipleJobsDetails) -> multipleJobsDetails .getJobs() .stream() .map(detail -> new JobStatusMessage( detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime())) .collect(Collectors.toList())); }
this.lastJobExecutionResult = JobClient.submitJobAndWait( actorSystem, flinkConfig, throw new ProgramInvocationException("The program execution failed: " + e.getMessage(), jobGraph.getJobID(), e);
/** * Submits a JobGraph detached. * @param jobGraph The JobGraph * @param classLoader User code class loader to deserialize the results and errors (may contain custom classes). * @return JobSubmissionResult * @throws ProgramInvocationException */ public JobSubmissionResult runDetached(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { waitForClusterToBeReady(); final ActorGateway jobManagerGateway; try { jobManagerGateway = getJobManagerGateway(); } catch (Exception e) { throw new ProgramInvocationException("Failed to retrieve the JobManager gateway.", jobGraph.getJobID(), e); } try { logAndSysout("Submitting Job with JobID: " + jobGraph.getJobID() + ". Returning after job submission."); JobClient.submitJobDetached( new AkkaJobManagerGateway(jobManagerGateway), flinkConfig, jobGraph, Time.milliseconds(timeout.toMillis()), classLoader); return new JobSubmissionResult(jobGraph.getJobID()); } catch (JobExecutionException e) { throw new ProgramInvocationException("The program execution failed: " + e.getMessage(), jobGraph.getJobID(), e); } }
@Test public void testSuccessfulProgramAfterFailure() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); runConnectedComponents(env); try { runKMeans(env); fail("This program execution should have failed."); } catch (JobExecutionException e) { assertTrue(e.getCause().getMessage().contains("Insufficient number of network buffers")); } runConnectedComponents(env); }
@Override public void run() { try { JobExecutionResult result = JobClient.submitJobAndWait( clientActorSystem, cluster.configuration(), cluster.highAvailabilityServices(), graph, timeout, false, getClass().getClassLoader()); resultPromise.success(result); } catch (Exception e) { // This was not expected... fail the test case resultPromise.failure(e); } } }
private static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); } }
private static ClusterClient<String> createClusterClient() throws Exception { final ClusterClient<String> clusterClient = mock(ClusterClient.class); when(clusterClient.listJobs()).thenReturn(CompletableFuture.completedFuture(Arrays.asList( new JobStatusMessage(new JobID(), "job1", JobStatus.RUNNING, 1L), new JobStatusMessage(new JobID(), "job2", JobStatus.CREATED, 1L), new JobStatusMessage(new JobID(), "job3", JobStatus.SUSPENDING, 3L), new JobStatusMessage(new JobID(), "job4", JobStatus.SUSPENDING, 2L), new JobStatusMessage(new JobID(), "job5", JobStatus.FINISHED, 3L) ))); return clusterClient; } }
@Override protected void testProgram() throws Exception { //test failing version try { executeTask(new FailingTestMapper(), 1); } catch (RuntimeException e) { //expected for collection execution if (!isCollectionExecution()) { Assert.fail(); } // for collection execution, no restarts. So, exception should be appended with 0. Assert.assertEquals(EXCEPTION_STRING + ":0", e.getMessage()); } catch (JobExecutionException e) { //expected for cluster execution if (isCollectionExecution()) { Assert.fail(); } // for cluster execution, one restart. So, exception should be appended with 1. Assert.assertEquals(EXCEPTION_STRING + ":1", e.getCause().getMessage()); } //test correct version executeTask(new TestMapper(), 0); }
private static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); } }
@Test public void testNullValues() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); env.getConfig().disableSysoutLogging(); DataSet<String> data = env.fromElements("hallo") .map(new MapFunction<String, String>() { @Override public String map(String value) throws Exception { return null; } }); data.writeAsText("/tmp/myTest", FileSystem.WriteMode.OVERWRITE); try { env.execute(); fail("this should fail due to null values."); } catch (JobExecutionException e) { assertNotNull(e.getCause()); assertTrue(e.getCause() instanceof NullPointerException); } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
public static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); }
@Test public void testListJobs() throws Exception { try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(new TestListJobsHandler())) { RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); try { CompletableFuture<Collection<JobStatusMessage>> jobDetailsFuture = restClusterClient.listJobs(); Collection<JobStatusMessage> jobDetails = jobDetailsFuture.get(); Iterator<JobStatusMessage> jobDetailsIterator = jobDetails.iterator(); JobStatusMessage job1 = jobDetailsIterator.next(); JobStatusMessage job2 = jobDetailsIterator.next(); Assert.assertNotEquals("The job status should not be equal.", job1.getJobState(), job2.getJobState()); } finally { restClusterClient.shutdown(); } } }
@Test public void testClusterClientList() throws Exception { Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, "localhost"); TestListActorGateway gateway = new TestListActorGateway(); TestClusterClient clusterClient = new TestClusterClient(config, gateway); try { CompletableFuture<Collection<JobStatusMessage>> jobDetailsFuture = clusterClient.listJobs(); Collection<JobStatusMessage> jobDetails = jobDetailsFuture.get(); Assert.assertTrue(gateway.messageArrived); Assert.assertEquals(2, jobDetails.size()); Iterator<JobStatusMessage> jobDetailsIterator = jobDetails.iterator(); JobStatusMessage job1 = jobDetailsIterator.next(); JobStatusMessage job2 = jobDetailsIterator.next(); Assert.assertNotEquals("The job statues should not be equal.", job1.getJobState(), job2.getJobState()); } finally { clusterClient.shutdown(); } }
@Test public void testInvalidTypeAccumulator() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); // Test Exception forwarding with faulty Accumulator implementation env.generateSequence(0, 10000) .map(new IncompatibleAccumulatorTypesMapper()) .map(new IncompatibleAccumulatorTypesMapper2()) .output(new DiscardingOutputFormat<>()); try { env.execute(); fail("Should have failed."); } catch (JobExecutionException e) { assertTrue("Root cause should be:", e.getCause() instanceof Exception); assertTrue("Root cause should be:", e.getCause().getCause() instanceof UnsupportedOperationException); } }