@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd.reason() instanceof org.apache.spark.Success$ && !taskEnd.taskInfo().speculative()) { Metrics metrics = new Metrics(taskEnd.taskMetrics(), taskEnd.taskInfo()); Integer jobId; synchronized (stageToJobId) { jobId = stageToJobId.get(taskEnd.stageId()); } // TODO: implement implicit AsyncRDDActions conversion instead of jc.monitor()? // TODO: how to handle stage failures? String clientId = getClientId(jobId); if (clientId != null) { protocol.sendMetrics(clientId, jobId, taskEnd.stageId(), taskEnd.taskInfo().taskId(), metrics); } } }
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Can not find job id for stage[" + stageId + "]."); } else { Map<Integer, List<Map.Entry<TaskMetrics, TaskInfo>>> jobMetrics = allJobMetrics.computeIfAbsent( jobId, k -> Maps.newHashMap()); List<Map.Entry<TaskMetrics, TaskInfo>> stageMetrics = jobMetrics.computeIfAbsent(stageId, k -> Lists.newLinkedList()); stageMetrics.add(new AbstractMap.SimpleEntry<>(taskEnd.taskMetrics(), taskEnd.taskInfo())); } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { super.onTaskEnd(taskEnd); String reason = taskEnd.reason().toString(); if (reason.equals("Success")) { sparkUIManager.taskEnd(taskEnd.stageId(), taskEnd.taskInfo().taskId()); } else if (reason.contains("stage cancelled")) { sparkUIManager.taskCancelled(taskEnd.stageId(), taskEnd.taskInfo().taskId()); } } });
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { try { String status = taskEnd.taskInfo().status(); .setStartTime(taskEnd.taskInfo().launchTime()) .setTaskId(String.valueOf(taskEnd.taskInfo().taskId())) .setStageId(String.valueOf(taskEnd.stageId())) .setStageAttemptId(String.valueOf(taskEnd.stageAttemptId())) .setExecutorHostname(String.valueOf(taskEnd.taskInfo().host())); tryToSet(() -> taskEventBuilder.setLocality(taskEnd.taskInfo().taskLocality().toString())); tryToSet(() -> taskEventBuilder.setType(taskEnd.taskType())); tryToSet(() -> taskEventBuilder.setAttemptNumber(taskEnd.taskInfo().attemptNumber())); tryToSet(() -> taskEventBuilder.setExecutorCpuTime(taskEnd.taskMetrics().executorCpuTime())); tryToSet(() -> taskEventBuilder.setExecutorDeserializeCpuTime(taskEnd.taskMetrics().executorDeserializeCpuTime())); tryToSet(() -> taskEventBuilder.setExecutorRunTime(taskEnd.taskMetrics().executorRunTime())); tryToSet(() -> taskEventBuilder.setJvmGcTime(taskEnd.taskMetrics().jvmGCTime())); tryToSet(() -> taskEventBuilder.setExecutorDeserializeTime(taskEnd.taskMetrics().executorDeserializeTime())); tryToSet(() -> taskEventBuilder.setResultSerializationTime(taskEnd.taskMetrics().resultSerializationTime())); tryToSet(() -> taskEventBuilder.setResultSize(taskEnd.taskMetrics().resultSize())); tryToSet(() -> taskEventBuilder.setPeakExecutionMemory(taskEnd.taskMetrics().peakExecutionMemory())); tryToSet(() -> taskEventBuilder.setDiskBytesSpilled(taskEnd.taskMetrics().diskBytesSpilled())); tryToSet(() -> taskEventBuilder.setMemoryBytesSpilled(taskEnd.taskMetrics().memoryBytesSpilled())); tryToSet(() -> taskEventBuilder.setShuffleReadRecords(taskEnd.taskMetrics().shuffleReadMetrics().recordsRead())); tryToSet(() -> taskEventBuilder.setShuffleReadFetchWaitTime(taskEnd.taskMetrics().shuffleReadMetrics().fetchWaitTime())); tryToSet(() -> taskEventBuilder.setShuffleReadLocalBytes(taskEnd.taskMetrics().shuffleReadMetrics().localBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadRemoteBytes(taskEnd.taskMetrics().shuffleReadMetrics().remoteBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadTotalBytes(taskEnd.taskMetrics().shuffleReadMetrics().totalBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadLocalBlocksFetched(taskEnd.taskMetrics().shuffleReadMetrics().localBlocksFetched()));
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); int stageAttemptId = taskEnd.stageAttemptId(); String stageIdentifier = stageId + "_" + stageAttemptId; Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Cannot find job id for stage[" + stageId + "]."); } else { Map<String, List<TaskMetrics>> jobMetrics = allJobMetrics.get(jobId); if (jobMetrics == null) { jobMetrics = Maps.newHashMap(); allJobMetrics.put(jobId, jobMetrics); } List<TaskMetrics> stageMetrics = jobMetrics.get(stageIdentifier); if (stageMetrics == null) { stageMetrics = Lists.newLinkedList(); jobMetrics.put(stageIdentifier, stageMetrics); } stageMetrics.add(taskEnd.taskMetrics()); } }
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Can not find job id for stage[" + stageId + "]."); } else { Map<Integer, List<TaskMetrics>> jobMetrics = allJobMetrics.get(jobId); if (jobMetrics == null) { jobMetrics = Maps.newHashMap(); allJobMetrics.put(jobId, jobMetrics); } List<TaskMetrics> stageMetrics = jobMetrics.get(stageId); if (stageMetrics == null) { stageMetrics = Lists.newLinkedList(); jobMetrics.put(stageId, stageMetrics); } stageMetrics.add(taskEnd.taskMetrics()); } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd != null && taskEnd.taskMetrics() != null && taskEnd.taskMetrics().outputMetrics() != null) { metrics.bytesWritten += taskEnd.taskMetrics().outputMetrics().bytesWritten(); metrics.recordsWritten += taskEnd.taskMetrics().outputMetrics().recordsWritten(); } }
@Override public void onTaskEnd(SparkListenerTaskEnd sparkListenerTaskEnd) { logDebug("Task '" + sparkListenerTaskEnd.taskInfo().id() + "' completed."); }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { Iterator<AccumulatorV2<?, ?>> iterator = taskEnd.taskMetrics().accumulators().iterator(); while (iterator.hasNext()) { AccumulatorV2 accumulator = iterator.next(); if (taskEnd.stageId() == 1 && accumulator.isRegistered() && accumulator.name().isDefined() && accumulator.name().get().equals("internal.metrics.shuffle.read.recordsRead")) { stageOneShuffleReadTaskRecordsCountMap.put(taskEnd.taskInfo().taskId(), (Long) accumulator.value()); } } } });
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); int stageAttemptId = taskEnd.stageAttemptId(); String stageIdentifier = stageId + "_" + stageAttemptId; Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Can not find job id for stage[" + stageId + "]."); } else { Map<String, List<TaskMetrics>> jobMetrics = allJobMetrics.get(jobId); if (jobMetrics == null) { jobMetrics = Maps.newHashMap(); allJobMetrics.put(jobId, jobMetrics); } List<TaskMetrics> stageMetrics = jobMetrics.get(stageIdentifier); if (stageMetrics == null) { stageMetrics = Lists.newLinkedList(); jobMetrics.put(stageIdentifier, stageMetrics); } stageMetrics.add(taskEnd.taskMetrics()); } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd != null && taskEnd.taskMetrics() != null && taskEnd.taskMetrics().outputMetrics() != null) { metrics.bytesWritten += taskEnd.taskMetrics().outputMetrics().bytesWritten(); metrics.recordsWritten += taskEnd.taskMetrics().outputMetrics().recordsWritten(); } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd.reason() instanceof org.apache.spark.Success$ && !taskEnd.taskInfo().speculative()) { Metrics metrics = new Metrics(taskEnd.taskMetrics()); Integer jobId; synchronized (stageToJobId) { jobId = stageToJobId.get(taskEnd.stageId()); } // TODO: implement implicit AsyncRDDActions conversion instead of jc.monitor()? // TODO: how to handle stage failures? String clientId = getClientId(jobId); if (clientId != null) { protocol.sendMetrics(clientId, jobId, taskEnd.stageId(), taskEnd.taskInfo().taskId(), metrics); } } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd.reason() instanceof org.apache.spark.Success$ && !taskEnd.taskInfo().speculative()) { Metrics metrics = new Metrics(taskEnd.taskMetrics()); Integer jobId; synchronized (stageToJobId) { jobId = stageToJobId.get(taskEnd.stageId()); } // TODO: implement implicit AsyncRDDActions conversion instead of jc.monitor()? // TODO: how to handle stage failures? String clientId = getClientId(jobId); if (clientId != null) { protocol.sendMetrics(clientId, jobId, taskEnd.stageId(), taskEnd.taskInfo().taskId(), metrics); } } }