/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
private void close(boolean delete) throws IOException { try { // send off set last segment, if we have not been closed before MemorySegment current = getCurrentSegment(); if (current != null) { writeSegment(current, getCurrentPositionInSegment()); } clear(); if (delete) { writer.closeAndDelete(); } else { writer.close(); } } finally { memManager.release(memory); } }
private void close(boolean delete) throws IOException { try { // send off set last segment, if we have not been closed before MemorySegment current = getCurrentSegment(); if (current != null) { writeSegment(current, getCurrentPositionInSegment()); } clear(); if (delete) { writer.closeAndDelete(); } else { writer.close(); } } finally { memManager.release(memory); } }
/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
private void close(boolean delete) throws IOException { try { // send off set last segment, if we have not been closed before MemorySegment current = getCurrentSegment(); if (current != null) { writeSegment(current, getCurrentPositionInSegment()); } clear(); if (delete) { writer.closeAndDelete(); } else { writer.close(); } } finally { memManager.release(memory); } }
/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
private void close(boolean delete) throws IOException { try { // send off set last segment, if we have not been closed before MemorySegment current = getCurrentSegment(); if (current != null) { writeSegment(current, getCurrentPositionInSegment()); } clear(); if (delete) { writer.closeAndDelete(); } else { writer.close(); } } finally { memManager.release(memory); } }
/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
private void spill() throws IOException { FileIOChannel.ID channel = ioManager.createChannel(); final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel); int numRecordBuffers = inMemoryBuffer.getNumRecordBuffers(); ArrayList<MemorySegment> segments = inMemoryBuffer.getRecordBufferSegments(); try { // spill in memory buffer in zero-copy. for (int i = 0; i < numRecordBuffers; i++) { writer.writeBlock(segments.get(i)); } LOG.info("here spill the reset buffer data with {} bytes", writer.getSize()); writer.close(); } catch (IOException e) { writer.closeAndDelete(); throw e; } spillSize += numRecordBuffers * segmentSize; channelIDs.add(new ChannelWithMeta( channel, inMemoryBuffer.getNumRecordBuffers(), inMemoryBuffer.getNumBytesInLastBuffer())); this.numRowsUntilThisChannel.add(numRows); inMemoryBuffer.reset(); }
/** * Closes this OutputView, closing the underlying writer and returning all memory segments. * * @return A list containing all memory segments originally supplied to this view. * @throws IOException Thrown, if the underlying writer could not be properly closed. */ public List<MemorySegment> close() throws IOException { // send off set last segment writeSegment(getCurrentSegment(), getCurrentPositionInSegment(), true); clear(); // close the writer and gather all segments final LinkedBlockingQueue<MemorySegment> queue = this.writer.getReturnQueue(); this.writer.close(); // re-collect all memory segments ArrayList<MemorySegment> list = new ArrayList<MemorySegment>(this.numSegments); for (int i = 0; i < this.numSegments; i++) { final MemorySegment m = queue.poll(); if (m == null) { // we get null if the queue is empty. that should not be the case if the reader was properly closed. throw new RuntimeException("ChannelWriterOutputView: MemorySegments have been taken from return queue by different actor."); } list.add(m); } return list; }
/** * Closes this OutputView, closing the underlying writer and returning all memory segments. * * @return A list containing all memory segments originally supplied to this view. * @throws IOException Thrown, if the underlying writer could not be properly closed. */ public List<MemorySegment> close() throws IOException { // send off set last segment writeSegment(getCurrentSegment(), getCurrentPositionInSegment(), true); clear(); // close the writer and gather all segments final LinkedBlockingQueue<MemorySegment> queue = this.writer.getReturnQueue(); this.writer.close(); // re-collect all memory segments ArrayList<MemorySegment> list = new ArrayList<MemorySegment>(this.numSegments); for (int i = 0; i < this.numSegments; i++) { final MemorySegment m = queue.poll(); if (m == null) { // we get null if the queue is empty. that should not be the case if the reader was properly closed. throw new RuntimeException("ChannelWriterOutputView: MemorySegments have been taken from return queue by different actor."); } list.add(m); } return list; }
/** * Closes this OutputView, closing the underlying writer and returning all memory segments. * * @return A list containing all memory segments originally supplied to this view. * @throws IOException Thrown, if the underlying writer could not be properly closed. */ public List<MemorySegment> close() throws IOException { // send off set last segment writeSegment(getCurrentSegment(), getCurrentPositionInSegment(), true); clear(); // close the writer and gather all segments final LinkedBlockingQueue<MemorySegment> queue = this.writer.getReturnQueue(); this.writer.close(); // re-collect all memory segments ArrayList<MemorySegment> list = new ArrayList<MemorySegment>(this.numSegments); for (int i = 0; i < this.numSegments; i++) { final MemorySegment m = queue.poll(); if (m == null) { // we get null if the queue is empty. that should not be the case if the reader was properly closed. throw new RuntimeException("ChannelWriterOutputView: MemorySegments have been taken from return queue by different actor."); } list.add(m); } return list; }
/** * Closes this OutputView, closing the underlying writer and returning all memory segments. * * @return A list containing all memory segments originally supplied to this view. * @throws IOException Thrown, if the underlying writer could not be properly closed. */ public List<MemorySegment> close() throws IOException { // send off set last segment writeSegment(getCurrentSegment(), getCurrentPositionInSegment(), true); clear(); // close the writer and gather all segments final LinkedBlockingQueue<MemorySegment> queue = this.writer.getReturnQueue(); this.writer.close(); // re-collect all memory segments ArrayList<MemorySegment> list = new ArrayList<MemorySegment>(this.numSegments); for (int i = 0; i < this.numSegments; i++) { final MemorySegment m = queue.poll(); if (m == null) { // we get null if the queue is empty. that should not be the case if the reader was properly closed. throw new RuntimeException("ChannelWriterOutputView: MemorySegments have been taken from return queue by different actor."); } list.add(m); } return list; }
/** * After build phase. * @return build spill return buffer, if have spilled, it returns the current write buffer, * because it was used all the time in build phase, so it can only be returned at this time. */ int finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. this.buildSideChannel.close(); this.probeSideBuffer = FileChannelUtil.createOutputView(ioAccess, probeChannelEnumerator.next(), compressionEnable, compressionCodecFactory, compressionBlockSize, memorySegmentSize); return 1; } else { return 0; } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
/** * Closes this OutputView, closing the underlying writer. And return number bytes in last * memory segment. */ @Override public int close() throws IOException { if (!writer.isClosed()) { int currentPositionInSegment = getCurrentPositionInSegment(); // write last segment writer.writeBlock(getCurrentSegment()); clear(); writer.getReturnQueue().clear(); this.writer.close(); return currentPositionInSegment; } return -1; }
/** * After build phase. * @return build spill return buffer, if have spilled, it returns the current write buffer, * because it was used all the time in build phase, so it can only be returned at this time. */ int finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. this.buildSideChannel.close(); this.probeSideBuffer = FileChannelUtil.createOutputView(ioAccess, probeChannelEnumerator.next(), context.compressionEnable(), context.compressionCodecFactory(), context.compressionBlockSize(), segmentSize); return 1; } else { return 0; } }