public OutputMetrics(TaskMetrics metrics) { this(metrics.outputMetrics().bytesWritten(), metrics.outputMetrics().recordsWritten()); }
public static void updateSparkBytesWrittenMetrics(Logger log, FileSystem fs, Path[] commitPaths) { AtomicLong bytesWritten = new AtomicLong(); Arrays.stream(commitPaths).parallel().forEach(path -> { try { bytesWritten.addAndGet(fs.getFileStatus(path).getLen()); } catch (IOException e) { log.debug("Unable to collect stats for file: " + path + " output metrics may be inaccurate", e); } }); if (bytesWritten.get() > 0) { TaskContext.get().taskMetrics().outputMetrics().setBytesWritten(bytesWritten.get()); } } }
public static void updateSparkRecordsWrittenMetrics(long numRows) { TaskContext taskContext = TaskContext.get(); if (taskContext != null && numRows > 0) { taskContext.taskMetrics().outputMetrics().setRecordsWritten(numRows); } }
bytesWritten += taskMetrics.outputMetrics().get().bytesWritten();
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd != null && taskEnd.taskMetrics() != null && taskEnd.taskMetrics().outputMetrics() != null) { metrics.bytesWritten += taskEnd.taskMetrics().outputMetrics().bytesWritten(); metrics.recordsWritten += taskEnd.taskMetrics().outputMetrics().recordsWritten(); } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd != null && taskEnd.taskMetrics() != null && taskEnd.taskMetrics().outputMetrics() != null) { metrics.bytesWritten += taskEnd.taskMetrics().outputMetrics().bytesWritten(); metrics.recordsWritten += taskEnd.taskMetrics().outputMetrics().recordsWritten(); } }
tryToSet(() -> taskEventBuilder.setInputRecords(taskEnd.taskMetrics().inputMetrics().recordsRead())); tryToSet(() -> taskEventBuilder.setInputBytes(taskEnd.taskMetrics().inputMetrics().bytesRead())); tryToSet(() -> taskEventBuilder.setOutputRecords(taskEnd.taskMetrics().outputMetrics().recordsWritten())); tryToSet(() -> taskEventBuilder.setOutputBytes(taskEnd.taskMetrics().outputMetrics().bytesWritten()));
tryToSet(() -> stageEventBuilder.setInputRecords(stageCompleted.stageInfo().taskMetrics().inputMetrics().recordsRead())); tryToSet(() -> stageEventBuilder.setInputBytes(stageCompleted.stageInfo().taskMetrics().inputMetrics().bytesRead())); tryToSet(() -> stageEventBuilder.setOutputRecords(stageCompleted.stageInfo().taskMetrics().outputMetrics().recordsWritten())); tryToSet(() -> stageEventBuilder.setOutputBytes(stageCompleted.stageInfo().taskMetrics().outputMetrics().bytesWritten()));