private void close(boolean deleteFile) throws IOException { try { clear(); if (deleteFile) { reader.closeAndDelete(); } else { reader.close(); } } finally { synchronized (memory) { memManager.release(memory); memory.clear(); } } }
@Override protected MemorySegment nextSegment(MemorySegment current) throws IOException { // use the buffer to send the next request if (requestsRemaining > 0) { requestsRemaining--; spilledBufferSource.readBlock(current); } else { emptyBufferTarget.add(current); } // get the next buffer either from the return queue, or the full buffer source if (spilledBuffersRemaining > 0) { spilledBuffersRemaining--; try { return spilledBufferSource.getReturnQueue().take(); } catch (InterruptedException e) { throw new RuntimeException("Read End was interrupted while waiting for spilled buffer.", e); } } else if (fullBufferSource.size() > 0) { return fullBufferSource.removeFirst(); } else { clear(); // delete the channel, if we had one if (spilledBufferSource != null) { spilledBufferSource.closeAndDelete(); } throw new EOFException(); } }
@Override protected MemorySegment nextSegment(MemorySegment current) throws IOException { // check for end-of-stream if (numBlocksRemaining <= 0) { reader.close(); throw new EOFException(); } // send a request first. if we have only a single segment, this same segment will be the one obtained in the next lines if (current != null) { sendReadRequest(current); } // get the next segment numBlocksRemaining--; return reader.getNextReturnedBlock(); }
public void seek(long position) throws IOException { final int block = MathUtils.checkedDownCast(position / segmentSize); final int positionInBlock = (int) (position % segmentSize); if (position < 0 || block >= numBlocksTotal || (block == numBlocksTotal - 1 && positionInBlock > sizeOfLastBlock)) { throw new IllegalArgumentException("Position is out of range"); } clear(); if (reader != null) { reader.close(); } reader = ioManager.createBlockChannelReader(channelId); if (block > 0) { reader.seekToPosition(((long) block) * segmentSize); } this.numBlocksRemaining = this.numBlocksTotal - block; this.numRequestsRemaining = numBlocksRemaining; for (int i = 0; i < memory.size(); i++) { sendReadRequest(memory.get(i)); } numBlocksRemaining--; seekInput(reader.getNextReturnedBlock(), positionInBlock, numBlocksRemaining == 0 ? sizeOfLastBlock : segmentSize); }
@Override public void closeAndDelete() throws IOException { reader.closeAndDelete(); } }
private void sendReadRequest(MemorySegment seg) throws IOException { if (numRequestsRemaining > 0) { reader.readBlock(seg); numRequestsRemaining--; } } }
final LinkedBlockingQueue<MemorySegment> queue = this.reader.getReturnQueue(); this.reader.close();
reader.readBlock(firstSeg); firstSeg = reader.getReturnQueue().take();
protected List<MemorySegment> readAllBuffers(FileIOChannel.ID id, int blockCount) throws IOException { // we are guaranteed to stay in memory ensureNumBuffersReturned(blockCount); LinkedBlockingQueue<MemorySegment> retSegments = new LinkedBlockingQueue<>(); BlockChannelReader<MemorySegment> reader = FileChannelUtil.createBlockChannelReader( ioManager, id, retSegments, compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize); for (int i = 0; i < blockCount; i++) { reader.readBlock(availableMemory.remove(availableMemory.size() - 1)); } reader.closeAndDelete(); final List<MemorySegment> buffers = new ArrayList<>(); retSegments.drainTo(buffers); return buffers; }
final long channelLength = reader.getSize();
final MemorySegment seg = this.reader.getNextReturnedBlock();
private void closeCurrentFileReader() throws IOException { if (fileReader != null) { fileReader.close(); fileReader = null; } }
private void newSpilledIterator(int channelID, long offset) throws IOException { ChannelWithMeta channel = channelIDs.get(channelID); currentChannelID = channelID; // close current reader first. closeCurrentFileReader(); // calculate segment number int segmentNum = (int) (offset / segmentSize); long seekPosition = segmentNum * segmentSize; // new reader. this.fileReader = ioManager.createBlockChannelReader(channel.getChannel()); if (offset > 0) { // seek to the beginning of that segment fileReader.seekToPosition(seekPosition); } ChannelReaderInputView inView = new HeaderlessChannelReaderInputView( fileReader, freeMemory, channel.getBlockCount() - segmentNum, channel.getNumBytesInLastBlock(), false, offset - seekPosition ); this.currentIterator = new PagedChannelReaderInputViewIterator<>( inView, null, binaryRowSerializer ); }
public void seek(long position) throws IOException { final int block = MathUtils.checkedDownCast(position / segmentSize); final int positionInBlock = (int) (position % segmentSize); if (position < 0 || block >= numBlocksTotal || (block == numBlocksTotal - 1 && positionInBlock > sizeOfLastBlock)) { throw new IllegalArgumentException("Position is out of range"); } clear(); if (reader != null) { reader.close(); } reader = ioManager.createBlockChannelReader(channelId); if (block > 0) { reader.seekToPosition(((long) block) * segmentSize); } this.numBlocksRemaining = this.numBlocksTotal - block; this.numRequestsRemaining = numBlocksRemaining; for (int i = 0; i < memory.size(); i++) { sendReadRequest(memory.get(i)); } numBlocksRemaining--; seekInput(reader.getNextReturnedBlock(), positionInBlock, numBlocksRemaining == 0 ? sizeOfLastBlock : segmentSize); }
private void forceDispose(List<MemorySegment> freeMemTarget) throws InterruptedException { // add the current segment final MemorySegment current = getCurrentSegment(); clear(); if (current != null) { freeMemTarget.add(current); } // add all remaining memory freeMemTarget.addAll(fullBufferSource); // add the segments with the requests issued but not returned for (int i = spilledBuffersRemaining - requestsRemaining; i > 0; --i) { freeMemTarget.add(emptyBufferTarget.take()); } if (spilledBufferSource != null) { try { spilledBufferSource.closeAndDelete(); } catch (Throwable t) { // do nothing } } } }
private void sendReadRequest(MemorySegment seg) throws IOException { if (numRequestsRemaining > 0) { reader.readBlock(seg); numRequestsRemaining--; } } }
final LinkedBlockingQueue<MemorySegment> queue = this.reader.getReturnQueue(); this.reader.close();
reader.readBlock(firstSeg); firstSeg = reader.getReturnQueue().take();