/** * Returns the client job ID for the given Spark job ID. * * This will only work for jobs monitored via JobContext#monitor(). Other jobs won't be * matched, and this method will return `None`. */ private String getClientId(Integer jobId) { for (Map.Entry<String, JobWrapper<?>> e : activeJobs.entrySet()) { for (JavaFutureAction<?> future : e.getValue().jobs) { if (future.jobIds().contains(jobId)) { return e.getKey(); } } } return null; }
private void monitorJob(JavaFutureAction<?> job, SparkCounters sparkCounters, Set<Integer> cachedRDDIds) { jobs.add(job); if (!jc.getMonitoredJobs().containsKey(req.id)) { jc.getMonitoredJobs().put(req.id, new CopyOnWriteArrayList<JavaFutureAction<?>>()); } jc.getMonitoredJobs().get(req.id).add(job); this.sparkCounters = sparkCounters; this.cachedRDDIds = cachedRDDIds; sparkJobId = job.jobIds().get(0); protocol.jobSubmitted(req.id, sparkJobId); }
int jobId = future.jobIds().get(0); LocalSparkJobStatus sparkJobStatus = new LocalSparkJobStatus( sc, jobId, jobMetricsListener, sparkCounters, plan.getCachedRDDIds(), future);
@Test public void takeAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.takeAsync(1); List<Integer> result = future.get(); assertEquals(1, result.size()); assertEquals((Integer) 1, result.get(0)); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
int jobId = future.jobIds().get(0); LocalSparkJobStatus sparkJobStatus = new LocalSparkJobStatus( sc, jobId, jobMetricsListener, sparkCounters, plan.getCachedRDDIds(), future);
@Test public void takeAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.takeAsync(1); List<Integer> result = future.get(); assertEquals(1, result.size()); assertEquals((Integer) 1, result.get(0)); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void takeAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.takeAsync(1); List<Integer> result = future.get(); assertEquals(1, result.size()); assertEquals((Integer) 1, result.get(0)); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void countAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.countAsync(); long count = future.get(); assertEquals(data.size(), count); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void countAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.countAsync(); long count = future.get(); assertEquals(data.size(), count); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void countAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.countAsync(); long count = future.get(); assertEquals(data.size(), count); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void collectAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.collectAsync(); List<Integer> result = future.get(); assertEquals(data, result); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void collectAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.collectAsync(); List<Integer> result = future.get(); assertEquals(data, result); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void collectAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.collectAsync(); List<Integer> result = future.get(); assertEquals(data, result); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
/** * Returns the client job ID for the given Spark job ID. * * This will only work for jobs monitored via JobContext#monitor(). Other jobs won't be * matched, and this method will return `None`. */ private String getClientId(Integer jobId) { for (Map.Entry<String, JobWrapper<?>> e : activeJobs.entrySet()) { for (JavaFutureAction<?> future : e.getValue().jobs) { if (future.jobIds().contains(jobId)) { return e.getKey(); } } } return null; }
/** * Returns the client job ID for the given Spark job ID. * * This will only work for jobs monitored via JobContext#monitor(). Other jobs won't be * matched, and this method will return `None`. */ private String getClientId(Integer jobId) { for (Map.Entry<String, JobWrapper<?>> e : activeJobs.entrySet()) { for (JavaFutureAction<?> future : e.getValue().jobs) { if (future.jobIds().contains(jobId)) { return e.getKey(); } } } return null; }
private void monitorJob(JavaFutureAction<?> job, SparkCounters sparkCounters, Set<Integer> cachedRDDIds) { jobs.add(job); if (!jc.getMonitoredJobs().containsKey(req.id)) { jc.getMonitoredJobs().put(req.id, new CopyOnWriteArrayList<JavaFutureAction<?>>()); } jc.getMonitoredJobs().get(req.id).add(job); this.sparkCounters = sparkCounters; this.cachedRDDIds = cachedRDDIds; protocol.jobSubmitted(req.id, job.jobIds().get(0)); }
private void monitorJob(JavaFutureAction<?> job, SparkCounters sparkCounters, Set<Integer> cachedRDDIds) { jobs.add(job); if (!jc.getMonitoredJobs().containsKey(req.id)) { jc.getMonitoredJobs().put(req.id, new CopyOnWriteArrayList<JavaFutureAction<?>>()); } jc.getMonitoredJobs().get(req.id).add(job); this.sparkCounters = sparkCounters; this.cachedRDDIds = cachedRDDIds; protocol.jobSubmitted(req.id, job.jobIds().get(0)); }