@Override public SortedDataFile<T> createFile(List<MemorySegment> writeMemory) throws IOException { int fileId = nextFileId++; String path = ExternalBlockShuffleUtils.generateSpillPath(partitionDataRootPath, fileId); return new BufferSortedDataFile<T>(ioManager.createChannel(new File(path)), fileId, serialize, ioManager, writeMemory, serializerManager, numBytesOut, numBuffersOut); } }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("Spilling {} bytes ({} buffers} for sub partition {} of {}.", spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("{}: Spilling {} bytes ({} buffers} for sub partition {} of {}.", parent.getOwningTaskName(), spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("{}: Spilling {} bytes ({} buffers} for sub partition {} of {}.", parent.getOwningTaskName(), spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
} else { this.writer = this.ioManager.createBlockChannelWriter(this.ioManager.createChannel());
} else { this.writer = this.ioManager.createBlockChannelWriter(this.ioManager.createChannel());
} else { this.writer = this.ioManager.createBlockChannelWriter(this.ioManager.createChannel());
} else { this.writer = this.ioManager.createBlockChannelWriter(this.ioManager.createChannel());
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spilalble view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = 0; // Spill all buffers for (int i = 0; i < numberOfBuffers; i++) { Buffer buffer = buffers.remove(); spilledBytes += buffer.getSize(); spillWriter.writeBlock(buffer); } LOG.debug("Spilling {} bytes for sub partition {} of {}.", spilledBytes, index, parent.getPartitionId()); return numberOfBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
private void spill() throws IOException { FileIOChannel.ID channel = ioManager.createChannel(); final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel); int numRecordBuffers = inMemoryBuffer.getNumRecordBuffers(); ArrayList<MemorySegment> segments = inMemoryBuffer.getRecordBufferSegments(); try { // spill in memory buffer in zero-copy. for (int i = 0; i < numRecordBuffers; i++) { writer.writeBlock(segments.get(i)); } LOG.info("here spill the reset buffer data with {} bytes", writer.getSize()); writer.close(); } catch (IOException e) { writer.closeAndDelete(); throw e; } spillSize += numRecordBuffers * segmentSize; channelIDs.add(new ChannelWithMeta( channel, inMemoryBuffer.getNumRecordBuffers(), inMemoryBuffer.getNumBytesInLastBuffer())); this.numRowsUntilThisChannel.add(numRows); inMemoryBuffer.reset(); }
fileWriters[i] = ioManager.createStreamFileWriter(ioManager.createChannel(new File(path))); bytesWritten[i] = 0;
recordsChannel = ioManager.createChannel(); keysChannel = ioManager.createChannel();
BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
recordsChannel = ioManager.createChannel(); keysChannel = ioManager.createChannel();
recordsChannel = ioManager.createChannel(); keysChannel = ioManager.createChannel();
BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
final FileIOChannel.ID mergedChannelID = ioManager.createChannel(); channelManager.addChannel(mergedChannelID); AbstractChannelWriterOutputView output = null;
int fileId) throws IOException, InterruptedException { final FileIOChannel.ID channel = ioManager.createChannel(new File(ExternalBlockShuffleUtils.generateMergePath(partitionDataRootPath, fileId))); ConcatPartitionedBufferSortedDataFile<T> writer = new ConcatPartitionedBufferSortedDataFile<T>( numberOfSubpartitions, channel, fileId, ioManager);