private boolean isSameAsPreviousProgress( Map<SparkStage, SparkStageProgress> progressMap, Map<SparkStage, SparkStageProgress> lastProgressMap) { if (lastProgressMap == null) { return false; } if (progressMap.isEmpty()) { return lastProgressMap.isEmpty(); } else { if (lastProgressMap.isEmpty()) { return false; } else { if (progressMap.size() != lastProgressMap.size()) { return false; } for (Map.Entry<SparkStage, SparkStageProgress> entry : progressMap.entrySet()) { if (!lastProgressMap.containsKey(entry.getKey()) || !progressMap.get(entry.getKey()).equals(lastProgressMap.get(entry.getKey()))) { return false; } } } } return true; }
@Override public boolean equals(Object obj) { if (obj instanceof SparkStageProgress) { SparkStageProgress other = (SparkStageProgress) obj; return getTotalTaskCount() == other.getTotalTaskCount() && getSucceededTaskCount() == other.getSucceededTaskCount() && getRunningTaskCount() == other.getRunningTaskCount() && getFailedTaskCount() == other.getFailedTaskCount(); } return false; }
private int getCompletedStages() { int completed = 0; SortedSet<SparkStage> keys = new TreeSet<SparkStage>(progressMap.keySet()); for (SparkStage stage : keys) { SparkStageProgress progress = progressMap.get(stage); final int complete = progress.getSucceededTaskCount(); final int total = progress.getTotalTaskCount(); if (total > 0 && complete == total) { completed++; } } return completed; }
private void getSparkJobInfo(SparkJobStatus sparkJobStatus) { try { stageIds = new ArrayList<Integer>(); int[] ids = sparkJobStatus.getStageIds(); if (ids != null) { for (int stageId : ids) { stageIds.add(stageId); } } Map<SparkStage, SparkStageProgress> progressMap = sparkJobStatus.getSparkStageProgress(); int sumTotal = 0; int sumComplete = 0; int sumFailed = 0; for (SparkStage s : progressMap.keySet()) { SparkStageProgress progress = progressMap.get(s); final int complete = progress.getSucceededTaskCount(); final int total = progress.getTotalTaskCount(); final int failed = progress.getFailedTaskCount(); sumTotal += total; sumComplete += complete; sumFailed += failed; } succeededTaskCount = sumComplete; totalTaskCount = sumTotal; failedTaskCount = sumFailed; } catch (Exception e) { LOG.error("Failed to get Spark job information", e); } }
private Map<SparkStage, SparkStageProgress> progressMap() { return new HashMap<SparkStage, SparkStageProgress>() {{ put(new SparkStage(1, 0), new SparkStageProgress(4, 3, 1, 0)); put(new SparkStage(3, 1), new SparkStageProgress(6, 4, 1, 1)); put(new SparkStage(9, 0), new SparkStageProgress(5, 5, 0, 0)); put(new SparkStage(10, 2), new SparkStageProgress(5, 3, 2, 0)); put(new SparkStage(15, 1), new SparkStageProgress(4, 3, 1, 0)); put(new SparkStage(15, 2), new SparkStageProgress(4, 4, 0, 0)); put(new SparkStage(20, 3), new SparkStageProgress(3, 1, 1, 1)); put(new SparkStage(21, 1), new SparkStageProgress(2, 2, 0, 0)); }}; }
protected int getStageMaxTaskCount(Map<SparkStage, SparkStageProgress> progressMap) { int stageMaxTasks = 0; for (SparkStageProgress progress : progressMap.values()) { int tasks = progress.getTotalTaskCount(); if (tasks > stageMaxTasks) { stageMaxTasks = tasks; } } return stageMaxTasks; }
for (String s : progressMap.keySet()) { SparkStageProgress progress = progressMap.get(s); final int complete = progress.getSucceededTaskCount(); final int total = progress.getTotalTaskCount(); final int failed = progress.getFailedTaskCount(); sumTotal += total; sumComplete += complete;
@Override public Map<String, SparkStageProgress> getSparkStageProgress() { Map<String, SparkStageProgress> stageProgresses = new HashMap<String, SparkStageProgress>(); for (int stageId : getStageIds()) { SparkStageInfo sparkStageInfo = getStageInfo(stageId); if (sparkStageInfo != null) { int runningTaskCount = sparkStageInfo.numActiveTasks(); int completedTaskCount = sparkStageInfo.numCompletedTasks(); int failedTaskCount = sparkStageInfo.numFailedTasks(); int totalTaskCount = sparkStageInfo.numTasks(); SparkStageProgress sparkStageProgress = new SparkStageProgress( totalTaskCount, completedTaskCount, runningTaskCount, failedTaskCount); stageProgresses.put(String.valueOf(sparkStageInfo.stageId()) + "_" + sparkStageInfo.currentAttemptId(), sparkStageProgress); } } return stageProgresses; }
@Override public double progressedPercentage() { SortedSet<SparkStage> keys = new TreeSet<SparkStage>(progressMap.keySet()); int sumTotal = 0; int sumComplete = 0; for (SparkStage stage : keys) { SparkStageProgress progress = progressMap.get(stage); final int complete = progress.getSucceededTaskCount(); final int total = progress.getTotalTaskCount(); sumTotal += total; sumComplete += complete; } double progress = (sumTotal == 0) ? 1.0f : (float) sumComplete / (float) sumTotal; return progress; }
protected int getTotalTaskCount(Map<SparkStage, SparkStageProgress> progressMap) { int totalTasks = 0; for (SparkStageProgress progress : progressMap.values()) { totalTasks += progress.getTotalTaskCount(); } return totalTasks; }
@Override public boolean equals(Object obj) { if (obj instanceof SparkStageProgress) { SparkStageProgress other = (SparkStageProgress) obj; return getTotalTaskCount() == other.getTotalTaskCount() && getSucceededTaskCount() == other.getSucceededTaskCount() && getRunningTaskCount() == other.getRunningTaskCount() && getFailedTaskCount() == other.getFailedTaskCount(); } return false; }
@Override public Map<String, SparkStageProgress> getSparkStageProgress() throws HiveException { Map<String, SparkStageProgress> stageProgresses = new HashMap<String, SparkStageProgress>(); for (int stageId : getStageIds()) { SparkStageInfo sparkStageInfo = getSparkStageInfo(stageId); if (sparkStageInfo != null && sparkStageInfo.name() != null) { int runningTaskCount = sparkStageInfo.numActiveTasks(); int completedTaskCount = sparkStageInfo.numCompletedTasks(); int failedTaskCount = sparkStageInfo.numFailedTasks(); int totalTaskCount = sparkStageInfo.numTasks(); SparkStageProgress sparkStageProgress = new SparkStageProgress( totalTaskCount, completedTaskCount, runningTaskCount, failedTaskCount); stageProgresses.put(String.valueOf(sparkStageInfo.stageId()) + "_" + sparkStageInfo.currentAttemptId(), sparkStageProgress); } } return stageProgresses; }
private boolean isSameAsPreviousProgress( Map<String, SparkStageProgress> progressMap, Map<String, SparkStageProgress> lastProgressMap) { if (lastProgressMap == null) { return false; } if (progressMap.isEmpty()) { return lastProgressMap.isEmpty(); } else { if (lastProgressMap.isEmpty()) { return false; } else { if (progressMap.size() != lastProgressMap.size()) { return false; } for (String key : progressMap.keySet()) { if (!lastProgressMap.containsKey(key) || !progressMap.get(key).equals(lastProgressMap.get(key))) { return false; } } } } return true; }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("TotalTasks: "); sb.append(getTotalTaskCount()); sb.append(" Succeeded: "); sb.append(getSucceededTaskCount()); sb.append(" Running: "); sb.append(getRunningTaskCount()); sb.append(" Failed: "); sb.append(getFailedTaskCount()); return sb.toString(); } }
@Override public Map<SparkStage, SparkStageProgress> getSparkStageProgress() { Map<SparkStage, SparkStageProgress> stageProgresses = new HashMap<SparkStage, SparkStageProgress>(); for (int stageId : getStageIds()) { SparkStageInfo sparkStageInfo = getStageInfo(stageId); if (sparkStageInfo != null) { int runningTaskCount = sparkStageInfo.numActiveTasks(); int completedTaskCount = sparkStageInfo.numCompletedTasks(); int failedTaskCount = sparkStageInfo.numFailedTasks(); int totalTaskCount = sparkStageInfo.numTasks(); SparkStageProgress sparkStageProgress = new SparkStageProgress( totalTaskCount, completedTaskCount, runningTaskCount, failedTaskCount); SparkStage stage = new SparkStage(sparkStageInfo.stageId(), sparkStageInfo.currentAttemptId()); stageProgresses.put(stage, sparkStageProgress); } } return stageProgresses; }
private boolean isSameAsPreviousProgress( Map<String, SparkStageProgress> progressMap, Map<String, SparkStageProgress> lastProgressMap) { if (lastProgressMap == null) { return false; } if (progressMap.isEmpty()) { return lastProgressMap.isEmpty(); } else { if (lastProgressMap.isEmpty()) { return false; } else { if (progressMap.size() != lastProgressMap.size()) { return false; } for (String key : progressMap.keySet()) { if (!lastProgressMap.containsKey(key) || !progressMap.get(key).equals(lastProgressMap.get(key))) { return false; } } } } return true; } }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("TotalTasks: "); sb.append(getTotalTaskCount()); sb.append(" Succeeded: "); sb.append(getSucceededTaskCount()); sb.append(" Running: "); sb.append(getRunningTaskCount()); sb.append(" Failed: "); sb.append(getFailedTaskCount()); return sb.toString(); } }
@Override public Map<SparkStage, SparkStageProgress> getSparkStageProgress() throws HiveException { List<SparkStageInfo> sparkStagesInfo = getSparkStagesInfo(); Map<SparkStage, SparkStageProgress> stageProgresses = new HashMap<SparkStage, SparkStageProgress>(); if (sparkStagesInfo != null) { for (SparkStageInfo sparkStageInfo : sparkStagesInfo) { if (sparkStageInfo != null && sparkStageInfo.name() != null) { int runningTaskCount = sparkStageInfo.numActiveTasks(); int completedTaskCount = sparkStageInfo.numCompletedTasks(); int failedTaskCount = sparkStageInfo.numFailedTasks(); int totalTaskCount = sparkStageInfo.numTasks(); SparkStageProgress sparkStageProgress = new SparkStageProgress(totalTaskCount, completedTaskCount, runningTaskCount, failedTaskCount); SparkStage stage = new SparkStage(sparkStageInfo.stageId(), sparkStageInfo.currentAttemptId()); stageProgresses.put(stage, sparkStageProgress); } } } return stageProgresses; }
@Override public List<List<String>> rows() { List<List<String>> progressRows = new ArrayList<>(); SortedSet<SparkStage> keys = new TreeSet<SparkStage>(progressMap.keySet()); for (SparkStage stage : keys) { SparkStageProgress progress = progressMap.get(stage); final int complete = progress.getSucceededTaskCount(); final int total = progress.getTotalTaskCount(); final int running = progress.getRunningTaskCount(); final int failed = progress.getFailedTaskCount(); SparkJobMonitor.StageState state = total > 0 ? SparkJobMonitor.StageState.PENDING : SparkJobMonitor.StageState.FINISHED; if (complete > 0 || running > 0 || failed > 0) { if (complete < total) { state = SparkJobMonitor.StageState.RUNNING; } else { state = SparkJobMonitor.StageState.FINISHED; } } String attempt = String.valueOf(stage.getAttemptId()); String stageName = "Stage-" +String.valueOf(stage.getStageId()); String nameWithProgress = getNameWithProgress(stageName, complete, total); final int pending = total - complete - running; progressRows.add(Arrays .asList(nameWithProgress, attempt, state.toString(), String.valueOf(total), String.valueOf(complete), String.valueOf(running), String.valueOf(pending), String.valueOf(failed), "")); } return progressRows; }
@Override public Map<String, SparkStageProgress> getSparkStageProgress() { Map<String, SparkStageProgress> stageProgresses = new HashMap<String, SparkStageProgress>(); for (int stageId : getStageIds()) { SparkStageInfo sparkStageInfo = getStageInfo(stageId); if (sparkStageInfo != null) { int runningTaskCount = sparkStageInfo.numActiveTasks(); int completedTaskCount = sparkStageInfo.numCompletedTasks(); int failedTaskCount = sparkStageInfo.numFailedTasks(); int totalTaskCount = sparkStageInfo.numTasks(); SparkStageProgress sparkStageProgress = new SparkStageProgress( totalTaskCount, completedTaskCount, runningTaskCount, failedTaskCount); stageProgresses.put(String.valueOf(sparkStageInfo.stageId()) + "_" + sparkStageInfo.currentAttemptId(), sparkStageProgress); } } return stageProgresses; }