@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd != null && taskEnd.taskMetrics() != null && taskEnd.taskMetrics().outputMetrics() != null) { metrics.bytesWritten += taskEnd.taskMetrics().outputMetrics().bytesWritten(); metrics.recordsWritten += taskEnd.taskMetrics().outputMetrics().recordsWritten(); } }
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Can not find job id for stage[" + stageId + "]."); } else { Map<Integer, List<Map.Entry<TaskMetrics, TaskInfo>>> jobMetrics = allJobMetrics.computeIfAbsent( jobId, k -> Maps.newHashMap()); List<Map.Entry<TaskMetrics, TaskInfo>> stageMetrics = jobMetrics.computeIfAbsent(stageId, k -> Lists.newLinkedList()); stageMetrics.add(new AbstractMap.SimpleEntry<>(taskEnd.taskMetrics(), taskEnd.taskInfo())); } }
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Can not find job id for stage[" + stageId + "]."); } else { Map<Integer, List<TaskMetrics>> jobMetrics = allJobMetrics.get(jobId); if (jobMetrics == null) { jobMetrics = Maps.newHashMap(); allJobMetrics.put(jobId, jobMetrics); } List<TaskMetrics> stageMetrics = jobMetrics.get(stageId); if (stageMetrics == null) { stageMetrics = Lists.newLinkedList(); jobMetrics.put(stageId, stageMetrics); } stageMetrics.add(taskEnd.taskMetrics()); } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd.reason() instanceof org.apache.spark.Success$ && !taskEnd.taskInfo().speculative()) { Metrics metrics = new Metrics(taskEnd.taskMetrics(), taskEnd.taskInfo()); Integer jobId; synchronized (stageToJobId) { jobId = stageToJobId.get(taskEnd.stageId()); } // TODO: implement implicit AsyncRDDActions conversion instead of jc.monitor()? // TODO: how to handle stage failures? String clientId = getClientId(jobId); if (clientId != null) { protocol.sendMetrics(clientId, jobId, taskEnd.stageId(), taskEnd.taskInfo().taskId(), metrics); } } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd != null && taskEnd.taskMetrics() != null && taskEnd.taskMetrics().outputMetrics() != null) { metrics.bytesWritten += taskEnd.taskMetrics().outputMetrics().bytesWritten(); metrics.recordsWritten += taskEnd.taskMetrics().outputMetrics().recordsWritten(); } }
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); int stageAttemptId = taskEnd.stageAttemptId(); String stageIdentifier = stageId + "_" + stageAttemptId; Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Cannot find job id for stage[" + stageId + "]."); } else { Map<String, List<TaskMetrics>> jobMetrics = allJobMetrics.get(jobId); if (jobMetrics == null) { jobMetrics = Maps.newHashMap(); allJobMetrics.put(jobId, jobMetrics); } List<TaskMetrics> stageMetrics = jobMetrics.get(stageIdentifier); if (stageMetrics == null) { stageMetrics = Lists.newLinkedList(); jobMetrics.put(stageIdentifier, stageMetrics); } stageMetrics.add(taskEnd.taskMetrics()); } }
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); int stageAttemptId = taskEnd.stageAttemptId(); String stageIdentifier = stageId + "_" + stageAttemptId; Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Can not find job id for stage[" + stageId + "]."); } else { Map<String, List<TaskMetrics>> jobMetrics = allJobMetrics.get(jobId); if (jobMetrics == null) { jobMetrics = Maps.newHashMap(); allJobMetrics.put(jobId, jobMetrics); } List<TaskMetrics> stageMetrics = jobMetrics.get(stageIdentifier); if (stageMetrics == null) { stageMetrics = Lists.newLinkedList(); jobMetrics.put(stageIdentifier, stageMetrics); } stageMetrics.add(taskEnd.taskMetrics()); } }
tryToSet(() -> taskEventBuilder.setType(taskEnd.taskType())); tryToSet(() -> taskEventBuilder.setAttemptNumber(taskEnd.taskInfo().attemptNumber())); tryToSet(() -> taskEventBuilder.setExecutorCpuTime(taskEnd.taskMetrics().executorCpuTime())); tryToSet(() -> taskEventBuilder.setExecutorDeserializeCpuTime(taskEnd.taskMetrics().executorDeserializeCpuTime())); tryToSet(() -> taskEventBuilder.setExecutorRunTime(taskEnd.taskMetrics().executorRunTime())); tryToSet(() -> taskEventBuilder.setJvmGcTime(taskEnd.taskMetrics().jvmGCTime())); tryToSet(() -> taskEventBuilder.setExecutorDeserializeTime(taskEnd.taskMetrics().executorDeserializeTime())); tryToSet(() -> taskEventBuilder.setResultSerializationTime(taskEnd.taskMetrics().resultSerializationTime())); tryToSet(() -> taskEventBuilder.setResultSize(taskEnd.taskMetrics().resultSize())); tryToSet(() -> taskEventBuilder.setPeakExecutionMemory(taskEnd.taskMetrics().peakExecutionMemory())); tryToSet(() -> taskEventBuilder.setDiskBytesSpilled(taskEnd.taskMetrics().diskBytesSpilled())); tryToSet(() -> taskEventBuilder.setMemoryBytesSpilled(taskEnd.taskMetrics().memoryBytesSpilled())); tryToSet(() -> taskEventBuilder.setShuffleReadRecords(taskEnd.taskMetrics().shuffleReadMetrics().recordsRead())); tryToSet(() -> taskEventBuilder.setShuffleReadFetchWaitTime(taskEnd.taskMetrics().shuffleReadMetrics().fetchWaitTime())); tryToSet(() -> taskEventBuilder.setShuffleReadLocalBytes(taskEnd.taskMetrics().shuffleReadMetrics().localBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadRemoteBytes(taskEnd.taskMetrics().shuffleReadMetrics().remoteBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadTotalBytes(taskEnd.taskMetrics().shuffleReadMetrics().totalBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadLocalBlocksFetched(taskEnd.taskMetrics().shuffleReadMetrics().localBlocksFetched())); tryToSet(() -> taskEventBuilder.setShuffleReadRemoteBlocksFetched(taskEnd.taskMetrics().shuffleReadMetrics().remoteBlocksFetched())); tryToSet(() -> taskEventBuilder.setShuffleReadTotalBlocksFetched(taskEnd.taskMetrics().shuffleReadMetrics().totalBlocksFetched())); tryToSet(() -> taskEventBuilder.setShuffleWriteShuffleRecords(taskEnd.taskMetrics().shuffleWriteMetrics().shuffleRecordsWritten())); tryToSet(() -> taskEventBuilder.setShuffleWriteShuffleTime(taskEnd.taskMetrics().shuffleWriteMetrics().shuffleWriteTime())); tryToSet(() -> taskEventBuilder.setShuffleWriteShuffleBytes(taskEnd.taskMetrics().shuffleWriteMetrics().shuffleBytesWritten())); tryToSet(() -> taskEventBuilder.setInputRecords(taskEnd.taskMetrics().inputMetrics().recordsRead())); tryToSet(() -> taskEventBuilder.setInputBytes(taskEnd.taskMetrics().inputMetrics().bytesRead())); tryToSet(() -> taskEventBuilder.setOutputRecords(taskEnd.taskMetrics().outputMetrics().recordsWritten())); tryToSet(() -> taskEventBuilder.setOutputBytes(taskEnd.taskMetrics().outputMetrics().bytesWritten()));
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd.reason() instanceof org.apache.spark.Success$ && !taskEnd.taskInfo().speculative()) { Metrics metrics = new Metrics(taskEnd.taskMetrics()); Integer jobId; synchronized (stageToJobId) { jobId = stageToJobId.get(taskEnd.stageId()); } // TODO: implement implicit AsyncRDDActions conversion instead of jc.monitor()? // TODO: how to handle stage failures? String clientId = getClientId(jobId); if (clientId != null) { protocol.sendMetrics(clientId, jobId, taskEnd.stageId(), taskEnd.taskInfo().taskId(), metrics); } } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd.reason() instanceof org.apache.spark.Success$ && !taskEnd.taskInfo().speculative()) { Metrics metrics = new Metrics(taskEnd.taskMetrics()); Integer jobId; synchronized (stageToJobId) { jobId = stageToJobId.get(taskEnd.stageId()); } // TODO: implement implicit AsyncRDDActions conversion instead of jc.monitor()? // TODO: how to handle stage failures? String clientId = getClientId(jobId); if (clientId != null) { protocol.sendMetrics(clientId, jobId, taskEnd.stageId(), taskEnd.taskInfo().taskId(), metrics); } } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { Iterator<AccumulatorV2<?, ?>> iterator = taskEnd.taskMetrics().accumulators().iterator(); while (iterator.hasNext()) { AccumulatorV2 accumulator = iterator.next(); if (taskEnd.stageId() == 1 && accumulator.isRegistered() && accumulator.name().isDefined() && accumulator.name().get().equals("internal.metrics.shuffle.read.recordsRead")) { stageOneShuffleReadTaskRecordsCountMap.put(taskEnd.taskInfo().taskId(), (Long) accumulator.value()); } } } });