Refine search
private void writeEnoughRecordsToTriggerSortBufferExpansionAndSpill() throws Exception { memoryManager.limit(UnsafeShuffleWriter.DEFAULT_INITIAL_SORT_BUFFER_SIZE * 16); final UnsafeShuffleWriter<Object, Object> writer = createWriter(false); final ArrayList<Product2<Object, Object>> dataToWrite = new ArrayList<>(); for (int i = 0; i < UnsafeShuffleWriter.DEFAULT_INITIAL_SORT_BUFFER_SIZE + 1; i++) { dataToWrite.add(new Tuple2<>(i, i)); } writer.write(dataToWrite.iterator()); writer.stop(true); readRecordsFromFile(); assertSpillFilesWereCleanedUp(); ShuffleWriteMetrics shuffleWriteMetrics = taskMetrics.shuffleWriteMetrics(); assertEquals(dataToWrite.size(), shuffleWriteMetrics.recordsWritten()); assertThat(taskMetrics.diskBytesSpilled(), greaterThan(0L)); assertThat(taskMetrics.diskBytesSpilled(), lessThan(mergedOutputFile.length())); assertThat(taskMetrics.memoryBytesSpilled(), greaterThan(0L)); assertEquals(mergedOutputFile.length(), shuffleWriteMetrics.bytesWritten()); }
@Override public Option<MapStatus> stop(boolean success) { try { taskContext.taskMetrics().incPeakExecutionMemory(getPeakMemoryUsedBytes()); if (stopping) { return Option.apply(null); } else { stopping = true; if (success) { if (mapStatus == null) { throw new IllegalStateException("Cannot call stop(true) without having called write()"); } return Option.apply(mapStatus); } else { return Option.apply(null); } } } finally { if (sorter != null) { // If sorter is non-null, then this implies that we called stop() in response to an error, // so we need to clean up memory and spill files created by the sorter sorter.cleanupResources(); } } } }
public Metrics(TaskMetrics metrics, TaskInfo taskInfo) { this( metrics.executorDeserializeTime(), TimeUnit.NANOSECONDS.toMillis(metrics.executorDeserializeCpuTime()), metrics.executorRunTime(), TimeUnit.NANOSECONDS.toMillis(metrics.executorCpuTime()), metrics.resultSize(), metrics.jvmGCTime(), metrics.resultSerializationTime(), metrics.memoryBytesSpilled(), metrics.diskBytesSpilled(), taskInfo.duration(), optionalInputMetric(metrics), optionalShuffleReadMetric(metrics), optionalShuffleWriteMetrics(metrics), optionalOutputMetrics(metrics)); }
tryToSet(() -> taskEventBuilder.setType(taskEnd.taskType())); tryToSet(() -> taskEventBuilder.setAttemptNumber(taskEnd.taskInfo().attemptNumber())); tryToSet(() -> taskEventBuilder.setExecutorCpuTime(taskEnd.taskMetrics().executorCpuTime())); tryToSet(() -> taskEventBuilder.setExecutorDeserializeCpuTime(taskEnd.taskMetrics().executorDeserializeCpuTime())); tryToSet(() -> taskEventBuilder.setExecutorRunTime(taskEnd.taskMetrics().executorRunTime())); tryToSet(() -> taskEventBuilder.setJvmGcTime(taskEnd.taskMetrics().jvmGCTime())); tryToSet(() -> taskEventBuilder.setExecutorDeserializeTime(taskEnd.taskMetrics().executorDeserializeTime())); tryToSet(() -> taskEventBuilder.setResultSerializationTime(taskEnd.taskMetrics().resultSerializationTime())); tryToSet(() -> taskEventBuilder.setResultSize(taskEnd.taskMetrics().resultSize())); tryToSet(() -> taskEventBuilder.setPeakExecutionMemory(taskEnd.taskMetrics().peakExecutionMemory())); tryToSet(() -> taskEventBuilder.setDiskBytesSpilled(taskEnd.taskMetrics().diskBytesSpilled())); tryToSet(() -> taskEventBuilder.setMemoryBytesSpilled(taskEnd.taskMetrics().memoryBytesSpilled())); tryToSet(() -> taskEventBuilder.setShuffleReadRecords(taskEnd.taskMetrics().shuffleReadMetrics().recordsRead())); tryToSet(() -> taskEventBuilder.setShuffleReadFetchWaitTime(taskEnd.taskMetrics().shuffleReadMetrics().fetchWaitTime())); tryToSet(() -> taskEventBuilder.setShuffleReadLocalBytes(taskEnd.taskMetrics().shuffleReadMetrics().localBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadRemoteBytes(taskEnd.taskMetrics().shuffleReadMetrics().remoteBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadTotalBytes(taskEnd.taskMetrics().shuffleReadMetrics().totalBytesRead())); tryToSet(() -> taskEventBuilder.setShuffleReadLocalBlocksFetched(taskEnd.taskMetrics().shuffleReadMetrics().localBlocksFetched())); tryToSet(() -> taskEventBuilder.setShuffleReadRemoteBlocksFetched(taskEnd.taskMetrics().shuffleReadMetrics().remoteBlocksFetched())); tryToSet(() -> taskEventBuilder.setShuffleReadTotalBlocksFetched(taskEnd.taskMetrics().shuffleReadMetrics().totalBlocksFetched())); tryToSet(() -> taskEventBuilder.setShuffleWriteShuffleRecords(taskEnd.taskMetrics().shuffleWriteMetrics().shuffleRecordsWritten())); tryToSet(() -> taskEventBuilder.setShuffleWriteShuffleTime(taskEnd.taskMetrics().shuffleWriteMetrics().shuffleWriteTime())); tryToSet(() -> taskEventBuilder.setShuffleWriteShuffleBytes(taskEnd.taskMetrics().shuffleWriteMetrics().shuffleBytesWritten())); tryToSet(() -> taskEventBuilder.setInputRecords(taskEnd.taskMetrics().inputMetrics().recordsRead())); tryToSet(() -> taskEventBuilder.setInputBytes(taskEnd.taskMetrics().inputMetrics().bytesRead())); tryToSet(() -> taskEventBuilder.setOutputRecords(taskEnd.taskMetrics().outputMetrics().recordsWritten())); tryToSet(() -> taskEventBuilder.setOutputBytes(taskEnd.taskMetrics().outputMetrics().bytesWritten()));
for (TaskMetrics taskMetrics : stageMetric) { if (taskMetrics != null) { executorDeserializeTime += taskMetrics.executorDeserializeTime(); executorRunTime += taskMetrics.executorRunTime(); resultSize += taskMetrics.resultSize(); jvmGCTime += taskMetrics.jvmGCTime(); resultSerializationTime += taskMetrics.resultSerializationTime(); memoryBytesSpilled += taskMetrics.memoryBytesSpilled(); diskBytesSpilled += taskMetrics.diskBytesSpilled(); if (!taskMetrics.inputMetrics().isEmpty()) { inputMetricExist = true; bytesRead += taskMetrics.inputMetrics().get().bytesRead(); Option<ShuffleReadMetrics> shuffleReadMetricsOption = taskMetrics.shuffleReadMetrics(); if (!shuffleReadMetricsOption.isEmpty()) { shuffleReadMetricExist = true; remoteBytesRead += shuffleReadMetricsOption.get().remoteBytesRead(); Option<ShuffleWriteMetrics> shuffleWriteMetricsOption = taskMetrics.shuffleWriteMetrics(); if (!shuffleWriteMetricsOption.isEmpty()) { shuffleWriteMetricExist = true;
/** * Increase the peak execution memory for current task. */ public void incPeakExecutionMemory(long size) { TaskContext.get().taskMetrics().incPeakExecutionMemory(size); }
public Metrics(TaskMetrics metrics) { this( metrics.executorDeserializeTime(), metrics.executorRunTime(), metrics.resultSize(), metrics.jvmGCTime(), metrics.resultSerializationTime(), metrics.memoryBytesSpilled(), metrics.diskBytesSpilled(), optionalInputMetric(metrics), optionalShuffleReadMetric(metrics), optionalShuffleWriteMetrics(metrics)); }
BypassMergeSortShuffleWriter( BlockManager blockManager, IndexShuffleBlockResolver shuffleBlockResolver, BypassMergeSortShuffleHandle<K, V> handle, int mapId, TaskContext taskContext, SparkConf conf) { // Use getSizeAsKb (not bytes) to maintain backwards compatibility if no units are provided this.fileBufferSize = (int) conf.getSizeAsKb("spark.shuffle.file.buffer", "32k") * 1024; this.transferToEnabled = conf.getBoolean("spark.file.transferTo", true); this.blockManager = blockManager; final ShuffleDependency<K, V, V> dep = handle.dependency(); this.mapId = mapId; this.shuffleId = dep.shuffleId(); this.partitioner = dep.partitioner(); this.numPartitions = partitioner.numPartitions(); this.writeMetrics = taskContext.taskMetrics().shuffleWriteMetrics(); this.serializer = dep.serializer(); this.shuffleBlockResolver = shuffleBlockResolver; }
public ShuffleWriteMetrics(TaskMetrics metrics) { this(metrics.shuffleWriteMetrics().shuffleBytesWritten(), metrics.shuffleWriteMetrics().shuffleWriteTime(), metrics.shuffleWriteMetrics().shuffleRecordsWritten()); }
@Test public void testDiskSpilledBytes() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false); } // We will have at-least 2 memory pages allocated because of rounding happening due to // integer division of pageSizeBytes and recordSize. assertTrue(sorter.getNumberOfAllocatedPages() >= 2); assertTrue(taskContext.taskMetrics().diskBytesSpilled() == 0); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); assertTrue(iter.spill() > 0); assertTrue(taskContext.taskMetrics().diskBytesSpilled() > 0); assertEquals(0, iter.spill()); // Even if we did not spill second time, the disk spilled bytes should still be non-zero assertTrue(taskContext.taskMetrics().diskBytesSpilled() > 0); sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
public InputMetrics(TaskMetrics metrics) { this(metrics.inputMetrics().bytesRead(), metrics.inputMetrics().recordsRead()); }
tryToSet(() -> stageEventBuilder.setExecutorCpuTime(stageCompleted.stageInfo().taskMetrics().executorCpuTime())); tryToSet(() -> stageEventBuilder.setExecutorDeserializeCpuTime(stageCompleted.stageInfo().taskMetrics().executorDeserializeCpuTime())); tryToSet(() -> stageEventBuilder.setExecutorRunTime(stageCompleted.stageInfo().taskMetrics().executorRunTime())); tryToSet(() -> stageEventBuilder.setJvmGcTime(stageCompleted.stageInfo().taskMetrics().jvmGCTime())); tryToSet(() -> stageEventBuilder.setExecutorDeserializeTime(stageCompleted.stageInfo().taskMetrics().executorDeserializeTime())); tryToSet(() -> stageEventBuilder.setResultSerializationTime(stageCompleted.stageInfo().taskMetrics().resultSerializationTime())); tryToSet(() -> stageEventBuilder.setResultSize(stageCompleted.stageInfo().taskMetrics().resultSize())); tryToSet(() -> stageEventBuilder.setPeakExecutionMemory(stageCompleted.stageInfo().taskMetrics().peakExecutionMemory())); tryToSet(() -> stageEventBuilder.setDiskBytesSpilled(stageCompleted.stageInfo().taskMetrics().diskBytesSpilled())); tryToSet(() -> stageEventBuilder.setMemoryBytesSpilled(stageCompleted.stageInfo().taskMetrics().memoryBytesSpilled())); tryToSet(() -> stageEventBuilder.setShuffleReadRecords(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().recordsRead())); tryToSet(() -> stageEventBuilder.setShuffleReadFetchWaitTime(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().fetchWaitTime())); tryToSet(() -> stageEventBuilder.setShuffleReadLocalBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().localBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadRemoteBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().remoteBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadTotalBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().totalBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadLocalBlocksFetched(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics() .localBlocksFetched())); tryToSet(() -> stageEventBuilder.setShuffleReadRemoteBlocksFetched(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics() .remoteBlocksFetched())); tryToSet(() -> stageEventBuilder.setShuffleReadTotalBlocksFetched(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics() .totalBlocksFetched())); tryToSet(() -> stageEventBuilder.setShuffleWriteShuffleRecords(stageCompleted.stageInfo().taskMetrics().shuffleWriteMetrics() .shuffleRecordsWritten())); tryToSet(() -> stageEventBuilder.setShuffleWriteShuffleTime(stageCompleted.stageInfo().taskMetrics().shuffleWriteMetrics().shuffleWriteTime())); tryToSet(() -> stageEventBuilder.setShuffleWriteShuffleBytes(stageCompleted.stageInfo().taskMetrics().shuffleWriteMetrics().shuffleBytesWritten())); tryToSet(() -> stageEventBuilder.setInputRecords(stageCompleted.stageInfo().taskMetrics().inputMetrics().recordsRead())); tryToSet(() -> stageEventBuilder.setInputBytes(stageCompleted.stageInfo().taskMetrics().inputMetrics().bytesRead())); tryToSet(() -> stageEventBuilder.setOutputRecords(stageCompleted.stageInfo().taskMetrics().outputMetrics().recordsWritten())); tryToSet(() -> stageEventBuilder.setOutputBytes(stageCompleted.stageInfo().taskMetrics().outputMetrics().bytesWritten()));
for (TaskMetrics taskMetrics : stageMetric) { if (taskMetrics != null) { executorDeserializeTime += taskMetrics.executorDeserializeTime(); executorRunTime += taskMetrics.executorRunTime(); resultSize += taskMetrics.resultSize(); jvmGCTime += taskMetrics.jvmGCTime(); resultSerializationTime += taskMetrics.resultSerializationTime(); memoryBytesSpilled += taskMetrics.memoryBytesSpilled(); diskBytesSpilled += taskMetrics.diskBytesSpilled(); if (!taskMetrics.inputMetrics().isEmpty()) { inputMetricExist = true; bytesRead += taskMetrics.inputMetrics().get().bytesRead(); if (!taskMetrics.outputMetrics().isEmpty()) { outputMetricExist = true; bytesWritten += taskMetrics.outputMetrics().get().bytesWritten(); Option<ShuffleReadMetrics> shuffleReadMetricsOption = taskMetrics.shuffleReadMetrics(); if (!shuffleReadMetricsOption.isEmpty()) { shuffleReadMetricExist = true; Option<ShuffleWriteMetrics> shuffleWriteMetricsOption = taskMetrics.shuffleWriteMetrics(); if (!shuffleWriteMetricsOption.isEmpty()) { shuffleWriteMetricExist = true;
/** * Increase the peak execution memory for current task. */ public void incPeakExecutionMemory(long size) { TaskContext.get().taskMetrics().incPeakExecutionMemory(size); }
public Metrics(TaskMetrics metrics) { this( metrics.executorDeserializeTime(), metrics.executorRunTime(), metrics.resultSize(), metrics.jvmGCTime(), metrics.resultSerializationTime(), metrics.memoryBytesSpilled(), metrics.diskBytesSpilled(), optionalInputMetric(metrics), optionalShuffleReadMetric(metrics), optionalShuffleWriteMetrics(metrics)); }
BypassMergeSortShuffleWriter( BlockManager blockManager, IndexShuffleBlockResolver shuffleBlockResolver, BypassMergeSortShuffleHandle<K, V> handle, int mapId, TaskContext taskContext, SparkConf conf) { // Use getSizeAsKb (not bytes) to maintain backwards compatibility if no units are provided this.fileBufferSize = (int) conf.getSizeAsKb("spark.shuffle.file.buffer", "32k") * 1024; this.transferToEnabled = conf.getBoolean("spark.file.transferTo", true); this.blockManager = blockManager; final ShuffleDependency<K, V, V> dep = handle.dependency(); this.mapId = mapId; this.shuffleId = dep.shuffleId(); this.partitioner = dep.partitioner(); this.numPartitions = partitioner.numPartitions(); this.writeMetrics = taskContext.taskMetrics().shuffleWriteMetrics(); this.serializer = dep.serializer(); this.shuffleBlockResolver = shuffleBlockResolver; }
private static ShuffleWriteMetrics optionalShuffleWriteMetrics(TaskMetrics metrics) { return (metrics.shuffleWriteMetrics() != null) ? new ShuffleWriteMetrics(metrics) : null; }
@Test public void testDiskSpilledBytes() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false); } // We will have at-least 2 memory pages allocated because of rounding happening due to // integer division of pageSizeBytes and recordSize. assertTrue(sorter.getNumberOfAllocatedPages() >= 2); assertTrue(taskContext.taskMetrics().diskBytesSpilled() == 0); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); assertTrue(iter.spill() > 0); assertTrue(taskContext.taskMetrics().diskBytesSpilled() > 0); assertEquals(0, iter.spill()); // Even if we did not spill second time, the disk spilled bytes should still be non-zero assertTrue(taskContext.taskMetrics().diskBytesSpilled() > 0); sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
private static InputMetrics optionalInputMetric(TaskMetrics metrics) { return (metrics.inputMetrics() != null) ? new InputMetrics(metrics) : null; }
private void writeEnoughRecordsToTriggerSortBufferExpansionAndSpill() throws Exception { memoryManager.limit(UnsafeShuffleWriter.DEFAULT_INITIAL_SORT_BUFFER_SIZE * 16); final UnsafeShuffleWriter<Object, Object> writer = createWriter(false); final ArrayList<Product2<Object, Object>> dataToWrite = new ArrayList<>(); for (int i = 0; i < UnsafeShuffleWriter.DEFAULT_INITIAL_SORT_BUFFER_SIZE + 1; i++) { dataToWrite.add(new Tuple2<>(i, i)); } writer.write(dataToWrite.iterator()); writer.stop(true); readRecordsFromFile(); assertSpillFilesWereCleanedUp(); ShuffleWriteMetrics shuffleWriteMetrics = taskMetrics.shuffleWriteMetrics(); assertEquals(dataToWrite.size(), shuffleWriteMetrics.recordsWritten()); assertThat(taskMetrics.diskBytesSpilled(), greaterThan(0L)); assertThat(taskMetrics.diskBytesSpilled(), lessThan(mergedOutputFile.length())); assertThat(taskMetrics.memoryBytesSpilled(), greaterThan(0L)); assertEquals(mergedOutputFile.length(), shuffleWriteMetrics.bytesWritten()); }