public SpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories) { this.nonSpanningWrapper = new NonSpanningWrapper(); this.spanningWrapper = new SpanningWrapper(tmpDirectories); }
@Override public void clear() { this.nonSpanningWrapper.clear(); this.spanningWrapper.clear(); }
@Override public boolean hasUnfinishedData() { return this.nonSpanningWrapper.remaining() > 0 || this.spanningWrapper.getNumGatheredBytes() > 0; }
this.spanningWrapper.initializeWithPartialRecord(this.nonSpanningWrapper, len); this.nonSpanningWrapper.clear(); return DeserializationResult.PARTIAL_RECORD; this.spanningWrapper.initializeWithPartialLength(this.nonSpanningWrapper); this.nonSpanningWrapper.clear(); return DeserializationResult.PARTIAL_RECORD; if (this.spanningWrapper.hasFullRecord()) { target.read(this.spanningWrapper.getInputView()); this.spanningWrapper.moveRemainderToNonSpanningDeserializer(this.nonSpanningWrapper); this.spanningWrapper.clear();
protected void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException { // set the length and copy what is available to the buffer this.recordLength = nextRecordLength; final int numBytesChunk = partial.remaining(); if (nextRecordLength > THRESHOLD_FOR_SPILLING) { // create a spilling channel and put the data there this.spillingChannel = createSpillingChannel(); ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk); FileUtils.writeCompletely(spillingChannel, toWrite); } else { // collect in memory ensureBufferCapacity(numBytesChunk); partial.segment.get(partial.position, buffer, 0, numBytesChunk); } this.accumulatedRecordBytes = numBytesChunk; }
@Override public void setNextBuffer(Buffer buffer) throws IOException { currentBuffer = buffer; int offset = buffer.getMemorySegmentOffset(); MemorySegment segment = buffer.getMemorySegment(); int numBytes = buffer.getSize(); // check if some spanning record deserialization is pending if (this.spanningWrapper.getNumGatheredBytes() > 0) { this.spanningWrapper.addNextChunkFromMemorySegment(segment, offset, numBytes); } else { this.nonSpanningWrapper.initializeFromMemorySegment(segment, offset, numBytes + offset); } }
segmentRemaining -= toPut; if (this.recordLength > THRESHOLD_FOR_SPILLING) { this.spillingChannel = createSpillingChannel(); ensureBufferCapacity(accumulatedRecordBytes + toCopy); segment.get(segmentPosition, buffer, this.accumulatedRecordBytes, toCopy);