private LocalHiveSparkClient(SparkConf sparkConf) { sc = new JavaSparkContext(sparkConf); jobMetricsListener = new JobMetricsListener(); sc.sc().listenerBus().addListener(jobMetricsListener); }
@Override public void cleanup() { jobMetricsListener.cleanup(jobId); if (cachedRDDIds != null) { for (Integer cachedRDDId: cachedRDDIds) { sparkContext.sc().unpersistRDD(cachedRDDId, false); } } }
@Override public SparkStatistics getSparkStatistics() { SparkStatisticsBuilder sparkStatisticsBuilder = new SparkStatisticsBuilder(); // add Hive operator level statistics. sparkStatisticsBuilder.add(sparkCounters); // add spark job metrics. Map<Integer, List<Map.Entry<TaskMetrics, TaskInfo>>> jobMetric = jobMetricsListener.getJobMetric(jobId); if (jobMetric == null) { return null; } MetricsCollection metricsCollection = new MetricsCollection(); Set<Integer> stageIds = jobMetric.keySet(); for (int stageId : stageIds) { List<Map.Entry<TaskMetrics, TaskInfo>> taskMetrics = jobMetric.get(stageId); for (Map.Entry<TaskMetrics, TaskInfo> taskMetric : taskMetrics) { Metrics metrics = new Metrics(taskMetric.getKey(), taskMetric.getValue()); metricsCollection.addMetrics(jobId, stageId, 0, metrics); } } Map<String, Long> flatJobMetric = SparkMetricsUtils.collectMetrics(metricsCollection .getAllMetrics()); for (Map.Entry<String, Long> entry : flatJobMetric.entrySet()) { sparkStatisticsBuilder.add(SparkStatisticsNames.SPARK_GROUP_NAME, entry.getKey(), Long.toString(entry.getValue())); } return sparkStatisticsBuilder.build(); }
@Override public void cleanup() { jobMetricsListener.cleanup(jobId); if (cachedRDDIds != null) { for (Integer cachedRDDId: cachedRDDIds) { sparkContext.sc().unpersistRDD(cachedRDDId, false); } } }
private LocalHiveSparkClient(SparkConf sparkConf, HiveConf hiveConf) throws FileNotFoundException, MalformedURLException { String regJar = null; // the registrator jar should already be in CP when not in test mode if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEST)) { String kryoReg = sparkConf.get("spark.kryo.registrator", ""); if (SparkClientUtilities.HIVE_KRYO_REG_NAME.equals(kryoReg)) { regJar = SparkClientUtilities.findKryoRegistratorJar(hiveConf); SparkClientUtilities.addJarToContextLoader(new File(regJar)); } } sc = new JavaSparkContext(sparkConf); if (regJar != null) { sc.addJar(regJar); } jobMetricsListener = new JobMetricsListener(); sc.sc().addSparkListener(jobMetricsListener); }
@Override public SparkStatistics getSparkStatistics() { SparkStatisticsBuilder sparkStatisticsBuilder = new SparkStatisticsBuilder(); // add Hive operator level statistics. sparkStatisticsBuilder.add(sparkCounters); // add spark job metrics. String jobIdentifier = "Spark Job[" + jobId + "] Metrics"; Map<Integer, List<TaskMetrics>> jobMetric = jobMetricsListener.getJobMetric(jobId); if (jobMetric == null) { return null; } MetricsCollection metricsCollection = new MetricsCollection(); Set<Integer> stageIds = jobMetric.keySet(); for (int stageId : stageIds) { List<TaskMetrics> taskMetrics = jobMetric.get(stageId); for (TaskMetrics taskMetric : taskMetrics) { Metrics metrics = new Metrics(taskMetric); metricsCollection.addMetrics(jobId, stageId, 0, metrics); } } Map<String, Long> flatJobMetric = SparkMetricsUtils.collectMetrics(metricsCollection .getAllMetrics()); for (Map.Entry<String, Long> entry : flatJobMetric.entrySet()) { sparkStatisticsBuilder.add(jobIdentifier, entry.getKey(), Long.toString(entry.getValue())); } return sparkStatisticsBuilder.build(); }
@Override public void cleanup() { jobMetricsListener.cleanup(jobId); if (cachedRDDIds != null) { for (Integer cachedRDDId: cachedRDDIds) { sparkContext.sc().unpersistRDD(cachedRDDId, false); } } }
private LocalHiveSparkClient(SparkConf sparkConf) { sc = new JavaSparkContext(sparkConf); jobMetricsListener = new JobMetricsListener(); sc.sc().listenerBus().addListener(jobMetricsListener); }
@Override public SparkStatistics getSparkStatistics() { SparkStatisticsBuilder sparkStatisticsBuilder = new SparkStatisticsBuilder(); // add Hive operator level statistics. sparkStatisticsBuilder.add(sparkCounters); // add spark job metrics. String jobIdentifier = "Spark Job[" + jobId + "] Metrics"; Map<String, List<TaskMetrics>> jobMetric = jobMetricsListener.getJobMetric(jobId); if (jobMetric == null) { return null; } Map<String, Long> flatJobMetric = combineJobLevelMetrics(jobMetric); for (Map.Entry<String, Long> entry : flatJobMetric.entrySet()) { sparkStatisticsBuilder.add(jobIdentifier, entry.getKey(), Long.toString(entry.getValue())); } return sparkStatisticsBuilder.build(); }