segmentRemaining -= toPut; if (this.recordLength > THRESHOLD_FOR_SPILLING) { this.spillingChannel = createSpillingChannel(); } else { ensureBufferCapacity(this.recordLength);
@SuppressWarnings("resource") private FileChannel createSpillingChannel() throws IOException { if (spillFile != null) { throw new IllegalStateException("Spilling file already exists."); } // try to find a unique file name for the spilling channel int maxAttempts = 10; for (int attempt = 0; attempt < maxAttempts; attempt++) { String directory = tempDirs[rnd.nextInt(tempDirs.length)]; spillFile = new File(directory, randomString(rnd) + ".inputchannel"); if (spillFile.createNewFile()) { return new RandomAccessFile(spillFile, "rw").getChannel(); } } throw new IOException( "Could not find a unique file channel name in '" + Arrays.toString(tempDirs) + "' for spilling large records during deserialization."); }
target.read(this.spanningWrapper.getInputView()); this.spanningWrapper.clear();
target.read(this.spanningWrapper.getInputView()); this.spanningWrapper.clear();
target.read(this.spanningWrapper.getInputView()); this.spanningWrapper.clear();
segmentRemaining -= toPut; if (this.recordLength > THRESHOLD_FOR_SPILLING) { this.spillingChannel = createSpillingChannel(); } else { ensureBufferCapacity(this.recordLength);
this.spillingChannel = createSpillingChannel(); ensureBufferCapacity(accumulatedRecordBytes + toCopy); segment.get(segmentPosition, buffer, this.accumulatedRecordBytes, toCopy);
@SuppressWarnings("resource") private FileChannel createSpillingChannel() throws IOException { if (spillFile != null) { throw new IllegalStateException("Spilling file already exists."); } // try to find a unique file name for the spilling channel int maxAttempts = 10; for (int attempt = 0; attempt < maxAttempts; attempt++) { String directory = tempDirs[rnd.nextInt(tempDirs.length)]; spillFile = new File(directory, randomString(rnd) + ".inputchannel"); if (spillFile.createNewFile()) { return new RandomAccessFile(spillFile, "rw").getChannel(); } } throw new IOException( "Could not find a unique file channel name in '" + Arrays.toString(tempDirs) + "' for spilling large records during deserialization."); }
private void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException { // set the length and copy what is available to the buffer this.recordLength = nextRecordLength; final int numBytesChunk = partial.remaining(); if (nextRecordLength > THRESHOLD_FOR_SPILLING) { // create a spilling channel and put the data there this.spillingChannel = createSpillingChannel(); ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk); FileUtils.writeCompletely(this.spillingChannel, toWrite); } else { // collect in memory ensureBufferCapacity(nextRecordLength); partial.segment.get(partial.position, buffer, 0, numBytesChunk); } this.accumulatedRecordBytes = numBytesChunk; }
private void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException { // set the length and copy what is available to the buffer this.recordLength = nextRecordLength; final int numBytesChunk = partial.remaining(); if (nextRecordLength > THRESHOLD_FOR_SPILLING) { // create a spilling channel and put the data there this.spillingChannel = createSpillingChannel(); ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk); this.spillingChannel.write(toWrite); } else { // collect in memory ensureBufferCapacity(numBytesChunk); partial.segment.get(partial.position, buffer, 0, numBytesChunk); } this.accumulatedRecordBytes = numBytesChunk; }
private void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException { // set the length and copy what is available to the buffer this.recordLength = nextRecordLength; final int numBytesChunk = partial.remaining(); if (nextRecordLength > THRESHOLD_FOR_SPILLING) { // create a spilling channel and put the data there this.spillingChannel = createSpillingChannel(); ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk); FileUtils.writeCompletely(this.spillingChannel, toWrite); } else { // collect in memory ensureBufferCapacity(nextRecordLength); partial.segment.get(partial.position, buffer, 0, numBytesChunk); } this.accumulatedRecordBytes = numBytesChunk; }
public SpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories) { this.nonSpanningWrapper = new NonSpanningWrapper(); this.spanningWrapper = new SpanningWrapper(tmpDirectories); }
public SpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories) { this.nonSpanningWrapper = new NonSpanningWrapper(); this.spanningWrapper = new SpanningWrapper(tmpDirectories); }
@Override public void clear() { this.nonSpanningWrapper.clear(); this.spanningWrapper.clear(); }
@Override public void clear() { this.nonSpanningWrapper.clear(); this.spanningWrapper.clear(); }
@Override public void clear() { this.nonSpanningWrapper.clear(); this.spanningWrapper.clear(); }
@SuppressWarnings("resource") private FileChannel createSpillingChannel() throws IOException { if (spillFile != null) { throw new IllegalStateException("Spilling file already exists."); } String directory = tempDirs[rnd.nextInt(tempDirs.length)]; spillFile = new File(directory, randomString(rnd) + ".inputchannel"); return new RandomAccessFile(spillFile, "rw").getChannel(); }
public SpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories) { this.nonSpanningWrapper = new NonSpanningWrapper(); this.spanningWrapper = new SpanningWrapper(tmpDirectories); }