public SparkStatistics build() { List<SparkStatisticGroup> statisticGroups = new LinkedList<SparkStatisticGroup>(); for (Map.Entry<String, List<SparkStatistic>> entry : statisticMap.entrySet()) { String groupName = entry.getKey(); List<SparkStatistic> statisticList = entry.getValue(); statisticGroups.add(new SparkStatisticGroup(groupName, statisticList)); } return new SparkStatistics(statisticGroups); }
private void logSparkStatistic(SparkStatistics sparkStatistic) { Iterator<SparkStatisticGroup> groupIterator = sparkStatistic.getStatisticGroups(); while (groupIterator.hasNext()) { SparkStatisticGroup group = groupIterator.next(); LOG.info(group.getGroupName()); Iterator<SparkStatistic> statisticIterator = group.getStatistics(); while (statisticIterator.hasNext()) { SparkStatistic statistic = statisticIterator.next(); LOG.info("\t" + statistic.getName() + ": " + statistic.getValue()); } } }
public static long getSparkStatisticAsLong(SparkStatisticGroup group, String name) { return Long.parseLong(group.getSparkStatistic(name).getValue()); } }
SparkStatistics(List<SparkStatisticGroup> statisticGroupsList) { for (SparkStatisticGroup group : statisticGroupsList) { statisticGroups.put(group.getGroupName(), group); } }
if (sparkStatisticGroup.containsSparkStatistic(SparkStatisticsNames.TASK_DURATION_TIME)) { metricsString += SparkStatisticsNames.TASK_DURATION_TIME + colon + SparkMetricsUtils.getSparkStatisticAsLong(sparkStatisticGroup, if (sparkStatisticGroup.containsSparkStatistic(SparkStatisticsNames.EXECUTOR_CPU_TIME)) { metricsString += SparkStatisticsNames.EXECUTOR_CPU_TIME + colon + SparkMetricsUtils.getSparkStatisticAsLong(sparkStatisticGroup, if (sparkStatisticGroup.containsSparkStatistic(SparkStatisticsNames.JVM_GC_TIME)) { metricsString += SparkStatisticsNames.JVM_GC_TIME + colon + SparkMetricsUtils.getSparkStatisticAsLong(sparkStatisticGroup, if (sparkStatisticGroup.containsSparkStatistic(SparkStatisticsNames.BYTES_READ) && sparkStatisticGroup.containsSparkStatistic(SparkStatisticsNames.RECORDS_READ)) { metricsString += SparkStatisticsNames.BYTES_READ + forwardSlash + SparkStatisticsNames.RECORDS_READ + colon + if (sparkStatisticGroup.containsSparkStatistic( SparkStatisticsNames.SHUFFLE_TOTAL_BYTES_READ) && sparkStatisticGroup.containsSparkStatistic( SparkStatisticsNames.SHUFFLE_RECORDS_READ)) { metricsString += SparkStatisticsNames.SHUFFLE_TOTAL_BYTES_READ + forwardSlash + if (sparkStatisticGroup.containsSparkStatistic( SparkStatisticsNames.SHUFFLE_BYTES_WRITTEN) && sparkStatisticGroup.containsSparkStatistic( SparkStatisticsNames.SHUFFLE_RECORDS_WRITTEN)) { metricsString += SparkStatisticsNames.SHUFFLE_BYTES_WRITTEN + forwardSlash +
@VisibleForTesting static String sparkStatisticsToString(SparkStatistics sparkStatistic, int sparkJobID) { StringBuilder sparkStatsString = new StringBuilder(); sparkStatsString.append("\n\n"); sparkStatsString.append(String.format("=====Spark Job[%d] Statistics=====", sparkJobID)); sparkStatsString.append("\n\n"); Iterator<SparkStatisticGroup> groupIterator = sparkStatistic.getStatisticGroups(); while (groupIterator.hasNext()) { SparkStatisticGroup group = groupIterator.next(); sparkStatsString.append(group.getGroupName()).append("\n"); Iterator<SparkStatistic> statisticIterator = group.getStatistics(); while (statisticIterator.hasNext()) { SparkStatistic statistic = statisticIterator.next(); sparkStatsString.append("\t").append(statistic.getName()).append(": ").append( statistic.getValue()).append("\n"); } } return sparkStatsString.toString(); }
public SparkStatistics build() { List<SparkStatisticGroup> statisticGroups = new LinkedList<SparkStatisticGroup>(); for (Map.Entry<String, List<SparkStatistic>> entry : statisticMap.entrySet()) { String groupName = entry.getKey(); List<SparkStatistic> statisitcList = entry.getValue(); statisticGroups.add(new SparkStatisticGroup(groupName, statisitcList)); } return new SparkStatistics(statisticGroups); }
private void logSparkStatistic(SparkStatistics sparkStatistic) { Iterator<SparkStatisticGroup> groupIterator = sparkStatistic.getStatisticGroups(); while (groupIterator.hasNext()) { SparkStatisticGroup group = groupIterator.next(); LOG.info(group.getGroupName()); Iterator<SparkStatistic> statisticIterator = group.getStatistics(); while (statisticIterator.hasNext()) { SparkStatistic statistic = statisticIterator.next(); LOG.info("\t" + statistic.getName() + ": " + statistic.getValue()); } } }
public SparkStatistics build() { List<SparkStatisticGroup> statisticGroups = new LinkedList<SparkStatisticGroup>(); for (Map.Entry<String, List<SparkStatistic>> entry : statisticMap.entrySet()) { String groupName = entry.getKey(); List<SparkStatistic> statisitcList = entry.getValue(); statisticGroups.add(new SparkStatisticGroup(groupName, statisitcList)); } return new SparkStatistics(statisticGroups); }