@Override public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { super.onStageCompleted(stageCompleted); sparkUIManager.endStage(stageCompleted.stageInfo().stageId()); }
@Override public void onStageCompleted(final SparkListenerStageCompleted stageCompleted) { TimeoutManager.getInstance().setLastEventTime(stageCompleted.stageInfo().stageId()); final StageInfo stageInfo = stageCompleted.stageInfo(); if (stageInfo.completionTime().isDefined() && stageInfo.submissionTime().isDefined()) { SparkJobTracker.recordStageTime(stageInfo, (long) stageInfo.completionTime().get() - (long) stageInfo.submissionTime().get()); } else { log.error("Stage completed without submission or completion time. Stage {}: {}", stageInfo.stageId(), stageInfo.name()); } SparkJobTracker.removeStageInfo(stageInfo); }
@Override public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { try { long submissionTime = stageCompleted.stageInfo().submissionTime().getOrElse(zeroLongScala); long completionTime = stageCompleted.stageInfo().completionTime().getOrElse(currentTimeLongScala); String name = stageCompleted.stageInfo().name(); String stageId = String.valueOf(stageCompleted.stageInfo().stageId()); String attemptId = String.valueOf(stageCompleted.stageInfo().attemptId()); int numTasks = stageCompleted.stageInfo().numTasks(); String status = getValOrNull(() -> stageCompleted.stageInfo().getStatusString()); tryToSet(() -> stageEventBuilder.setExecutorCpuTime(stageCompleted.stageInfo().taskMetrics().executorCpuTime())); tryToSet(() -> stageEventBuilder.setExecutorDeserializeCpuTime(stageCompleted.stageInfo().taskMetrics().executorDeserializeCpuTime())); tryToSet(() -> stageEventBuilder.setExecutorRunTime(stageCompleted.stageInfo().taskMetrics().executorRunTime())); tryToSet(() -> stageEventBuilder.setJvmGcTime(stageCompleted.stageInfo().taskMetrics().jvmGCTime())); tryToSet(() -> stageEventBuilder.setExecutorDeserializeTime(stageCompleted.stageInfo().taskMetrics().executorDeserializeTime())); tryToSet(() -> stageEventBuilder.setResultSerializationTime(stageCompleted.stageInfo().taskMetrics().resultSerializationTime())); tryToSet(() -> stageEventBuilder.setResultSize(stageCompleted.stageInfo().taskMetrics().resultSize())); tryToSet(() -> stageEventBuilder.setPeakExecutionMemory(stageCompleted.stageInfo().taskMetrics().peakExecutionMemory())); tryToSet(() -> stageEventBuilder.setDiskBytesSpilled(stageCompleted.stageInfo().taskMetrics().diskBytesSpilled())); tryToSet(() -> stageEventBuilder.setMemoryBytesSpilled(stageCompleted.stageInfo().taskMetrics().memoryBytesSpilled())); tryToSet(() -> stageEventBuilder.setShuffleReadRecords(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().recordsRead())); tryToSet(() -> stageEventBuilder.setShuffleReadFetchWaitTime(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().fetchWaitTime())); tryToSet(() -> stageEventBuilder.setShuffleReadLocalBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().localBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadRemoteBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().remoteBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadTotalBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().totalBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadLocalBlocksFetched(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics() .localBlocksFetched())); tryToSet(() -> stageEventBuilder.setShuffleReadRemoteBlocksFetched(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics()