public ShuffleWriteMetrics(TaskMetrics metrics) { this(metrics.shuffleWriteMetrics().shuffleBytesWritten(), metrics.shuffleWriteMetrics().shuffleWriteTime(), metrics.shuffleWriteMetrics().shuffleRecordsWritten()); }
public ShuffleReadMetrics(TaskMetrics metrics) { this(metrics.shuffleReadMetrics().remoteBlocksFetched(), metrics.shuffleReadMetrics().localBlocksFetched(), metrics.shuffleReadMetrics().fetchWaitTime(), metrics.shuffleReadMetrics().remoteBytesRead(), metrics.shuffleReadMetrics().localBytesRead(), metrics.shuffleReadMetrics().remoteBytesReadToDisk(), metrics.shuffleReadMetrics().recordsRead()); }
public InputMetrics(TaskMetrics metrics) { this(metrics.inputMetrics().bytesRead(), metrics.inputMetrics().recordsRead()); }
public OutputMetrics(TaskMetrics metrics) { this(metrics.outputMetrics().bytesWritten(), metrics.outputMetrics().recordsWritten()); }
private static ShuffleWriteMetrics optionalShuffleWriteMetrics(TaskMetrics metrics) { return (metrics.shuffleWriteMetrics() != null) ? new ShuffleWriteMetrics(metrics) : null; }
@Override public void write(byte[] b) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b); writeMetrics.incWriteTime(System.nanoTime() - startTime); }
public static void updateSparkRecordsWrittenMetrics(long numRows) { TaskContext taskContext = TaskContext.get(); if (taskContext != null && numRows > 0) { taskContext.taskMetrics().outputMetrics().setRecordsWritten(numRows); } }
private static ShuffleReadMetrics optionalShuffleReadMetric(TaskMetrics metrics) { return (metrics.shuffleReadMetrics() != null) ? new ShuffleReadMetrics(metrics) : null; }
private static InputMetrics optionalInputMetric(TaskMetrics metrics) { return (metrics.inputMetrics() != null) ? new InputMetrics(metrics) : null; }
private static OutputMetrics optionalOutputMetrics(TaskMetrics metrics) { return (metrics.outputMetrics() != null) ? new OutputMetrics(metrics) : null; }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd != null && taskEnd.taskMetrics() != null && taskEnd.taskMetrics().outputMetrics() != null) { metrics.bytesWritten += taskEnd.taskMetrics().outputMetrics().bytesWritten(); metrics.recordsWritten += taskEnd.taskMetrics().outputMetrics().recordsWritten(); } }
@Override public void write(byte[] b) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b); writeMetrics.incWriteTime(System.nanoTime() - startTime); }
@Override public void close() throws IOException { final long startTime = System.nanoTime(); outputStream.close(); writeMetrics.incWriteTime(System.nanoTime() - startTime); } }
@Override public void write(byte[] b, int off, int len) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b, off, len); writeMetrics.incWriteTime(System.nanoTime() - startTime); }
@Override public void write(int b) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b); writeMetrics.incWriteTime(System.nanoTime() - startTime); }
@Override public void write(byte[] b) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b); writeMetrics.incWriteTime(System.nanoTime() - startTime); }
@Override public void write(byte[] b, int off, int len) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b, off, len); writeMetrics.incWriteTime(System.nanoTime() - startTime); }
@Override public void close() throws IOException { final long startTime = System.nanoTime(); outputStream.close(); writeMetrics.incWriteTime(System.nanoTime() - startTime); } }
@Override public void flush() throws IOException { final long startTime = System.nanoTime(); outputStream.flush(); writeMetrics.incWriteTime(System.nanoTime() - startTime); }
@Override public void flush() throws IOException { final long startTime = System.nanoTime(); outputStream.flush(); writeMetrics.incWriteTime(System.nanoTime() - startTime); }