@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
private boolean cancelJob(JobWrapper<?> job) { boolean cancelled = false; for (JavaFutureAction<?> action : job.jobs) { cancelled |= action.cancel(true); } return cancelled | (job.future != null && job.future.cancel(true)); }
@Test public void testAsyncActionCancellation() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> { Thread.sleep(10000); // To ensure that the job won't finish before it's cancelled. }); future.cancel(true); assertTrue(future.isCancelled()); assertTrue(future.isDone()); try { future.get(2000, TimeUnit.MILLISECONDS); fail("Expected future.get() for cancelled job to throw CancellationException"); } catch (CancellationException ignored) { // pass } }
/** * Returns the client job ID for the given Spark job ID. * * This will only work for jobs monitored via JobContext#monitor(). Other jobs won't be * matched, and this method will return `None`. */ private String getClientId(Integer jobId) { for (Map.Entry<String, JobWrapper<?>> e : activeJobs.entrySet()) { for (JavaFutureAction<?> future : e.getValue().jobs) { if (future.jobIds().contains(jobId)) { return e.getKey(); } } } return null; }
future.get(); completed++; LOG.debug("Client job {}: {} of {} Spark jobs finished.",
@Test public void testAsyncActionCancellation() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> { Thread.sleep(10000); // To ensure that the job won't finish before it's cancelled. }); future.cancel(true); assertTrue(future.isCancelled()); assertTrue(future.isDone()); try { future.get(2000, TimeUnit.MILLISECONDS); fail("Expected future.get() for cancelled job to throw CancellationException"); } catch (CancellationException ignored) { // pass } }
private void monitorJob(JavaFutureAction<?> job, SparkCounters sparkCounters, Set<Integer> cachedRDDIds) { jobs.add(job); if (!jc.getMonitoredJobs().containsKey(req.id)) { jc.getMonitoredJobs().put(req.id, new CopyOnWriteArrayList<JavaFutureAction<?>>()); } jc.getMonitoredJobs().get(req.id).add(job); this.sparkCounters = sparkCounters; this.cachedRDDIds = cachedRDDIds; sparkJobId = job.jobIds().get(0); protocol.jobSubmitted(req.id, sparkJobId); }
@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Override public JobExecutionStatus getState() { SparkJobInfo sparkJobInfo = getJobInfo(); // For spark job with empty source data, it's not submitted actually, so we would never // receive JobStart/JobEnd event in JobStateListener, use JavaFutureAction to get current // job state. if (sparkJobInfo == null && future.isDone()) { try { future.get(); } catch (Exception e) { LOG.error("Failed to run job " + jobId, e); return JobExecutionStatus.FAILED; } return JobExecutionStatus.SUCCEEDED; } return sparkJobInfo == null ? null : sparkJobInfo.status(); }
@Test public void testAsyncActionCancellation() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> { Thread.sleep(10000); // To ensure that the job won't finish before it's cancelled. }); future.cancel(true); assertTrue(future.isCancelled()); assertTrue(future.isDone()); try { future.get(2000, TimeUnit.MILLISECONDS); fail("Expected future.get() for cancelled job to throw CancellationException"); } catch (CancellationException ignored) { // pass } }
int jobId = future.jobIds().get(0); LocalSparkJobStatus sparkJobStatus = new LocalSparkJobStatus( sc, jobId, jobMetricsListener, sparkCounters, plan.getCachedRDDIds(), future);
private boolean cancelJob(JobWrapper<?> job) { boolean cancelled = false; for (JavaFutureAction<?> action : job.jobs) { cancelled |= action.cancel(true); } return cancelled | (job.future != null && job.future.cancel(true)); }
@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Override public JobExecutionStatus getState() { SparkJobInfo sparkJobInfo = getJobInfo(); // For spark job with empty source data, it's not submitted actually, so we would never // receive JobStart/JobEnd event in JobStateListener, use JavaFutureAction to get current // job state. if (sparkJobInfo == null && future.isDone()) { try { future.get(); } catch (Exception e) { LOG.error("Failed to run job " + jobId, e); return JobExecutionStatus.FAILED; } return JobExecutionStatus.SUCCEEDED; } return sparkJobInfo == null ? null : sparkJobInfo.status(); }
int jobId = future.jobIds().get(0); LocalSparkJobStatus sparkJobStatus = new LocalSparkJobStatus( sc, jobId, jobMetricsListener, sparkCounters, plan.getCachedRDDIds(), future);