@Override public BarrierBuffer createBarrierHandler(InputGate gate) throws IOException{ return new BarrierBuffer(gate, new BufferSpiller(ioManager, PAGE_SIZE)); }
private BufferOrEventSequence rollOver(boolean newBuffer) throws IOException { if (bytesWritten == 0) { return null; } ByteBuffer buf; if (newBuffer) { buf = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); buf.order(ByteOrder.LITTLE_ENDIAN); } else { buf = readBuffer; } // create a reader for the spilled data currentChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize); // create ourselves a new spill file createSpillingChannel(); bytesWritten = 0L; return seq; }
/** * The BufferOrEventSequence returned by this method is safe for concurrent consumption with * any previously returned sequence. * * @return The readable sequence of spilled buffers and events, or 'null', if nothing was added. * @throws IOException Thrown, if the readable sequence could not be created, or no new spill * file could be created. */ @Override public BufferOrEventSequence rollOverWithoutReusingResources() throws IOException { return rollOver(true); }
@After public void cleanupSpiller() throws IOException { if (spiller != null) { spiller.close(); assertFalse(spiller.getCurrentChannel().isOpen()); assertFalse(spiller.getCurrentSpillFile().exists()); } checkNoTempFilesRemain(); }
/** * Tests that the static HEADER_SIZE field has valid header size. */ @Test public void testHeaderSizeStaticField() throws Exception { int size = 13; BufferOrEvent boe = generateRandomBuffer(size, 0); spiller.add(boe); assertEquals( "Changed the header format, but did not adjust the HEADER_SIZE field", BufferSpiller.HEADER_SIZE + size, spiller.getBytesBlocked()); }
currentBuffered = bufferSpiller.rollOver(); if (currentBuffered != null) { currentBuffered.open(); BufferSpiller.SpilledBufferOrEventSequence bufferedNow = bufferSpiller.rollOverWithNewBuffer(); if (bufferedNow != null) { bufferedNow.open();
@Override public void cleanup() throws IOException { bufferSpiller.close(); if (currentBuffered != null) { currentBuffered.cleanup(); } for (BufferSpiller.SpilledBufferOrEventSequence seq : queuedBuffered) { seq.cleanup(); } queuedBuffered.clear(); numQueuedBytes = 0L; }
if (isBlocked(next.getChannelIndex())) { bufferSpiller.add(next); checkSizeLimit();
private void checkSizeLimit() throws Exception { if (maxBufferedBytes > 0 && (numQueuedBytes + bufferSpiller.getBytesWritten()) > maxBufferedBytes) { // exceeded our limit - abort this checkpoint LOG.info("Checkpoint {} aborted because alignment volume limit ({} bytes) exceeded", currentCheckpointId, maxBufferedBytes); releaseBlocksAndResetBarriers(); notifyAbort(currentCheckpointId, new AlignmentLimitExceededException(maxBufferedBytes)); } }
@Before public void createSpiller() throws IOException { spiller = new BufferSpiller(ioManager, PAGE_SIZE); }
/** * Creates a new buffer spiller, spilling to one of the I/O manager's temp directories. * * @param ioManager The I/O manager for access to the temp directories. * @param pageSize The page size used to re-create spilled buffers. * @throws IOException Thrown if the temp files for spilling cannot be initialized. */ public BufferSpiller(IOManager ioManager, int pageSize) throws IOException { this.pageSize = pageSize; this.readBuffer = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); this.readBuffer.order(ByteOrder.LITTLE_ENDIAN); this.headBuffer = ByteBuffer.allocateDirect(16); this.headBuffer.order(ByteOrder.LITTLE_ENDIAN); File[] tempDirs = ioManager.getSpillingDirectories(); this.tempDir = tempDirs[DIRECTORY_INDEX.getAndIncrement() % tempDirs.length]; byte[] rndBytes = new byte[32]; ThreadLocalRandom.current().nextBytes(rndBytes); this.spillFilePrefix = StringUtils.byteToHexString(rndBytes) + '.'; // prepare for first contents createSpillingChannel(); }
/** * NOTE: The BufferOrEventSequences created by this method all reuse the same reading memory * (to reduce overhead) and can consequently not be read concurrently with each other. * * <p>To create a sequence that can be read concurrently with the previous BufferOrEventSequence, * use the {@link #rollOverWithoutReusingResources()} ()} method. * * @return The readable sequence of spilled buffers and events, or 'null', if nothing was added. * @throws IOException Thrown, if the readable sequence could not be created, or no new spill * file could be created. */ @Override public BufferOrEventSequence rollOverReusingResources() throws IOException { return rollOver(false); }
barrierHandler = new BarrierBuffer(inputGate, new CachedBufferBlocker(inputGate.getPageSize()), maxAlign); } else { barrierHandler = new BarrierBuffer(inputGate, new BufferSpiller(ioManager, inputGate.getPageSize()), maxAlign);
private SpilledBufferOrEventSequence rollOverInternal(boolean newBuffer) throws IOException { if (bytesWritten == 0) { return null; } ByteBuffer buf; if (newBuffer) { buf = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); buf.order(ByteOrder.LITTLE_ENDIAN); } else { buf = readBuffer; } // create a reader for the spilled data currentChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize); // create ourselves a new spill file createSpillingChannel(); bytesWritten = 0L; return seq; }
/** * The BufferOrEventSequence returned by this method is safe for concurrent consumption with * any previously returned sequence. * * @return The readable sequence of spilled buffers and events, or 'null', if nothing was added. * @throws IOException Thrown, if the readable sequence could not be created, or no new spill * file could be created. */ @Override public BufferOrEventSequence rollOverWithoutReusingResources() throws IOException { return rollOver(true); }
new BarrierGenerator[] { new CountBarrier(100000), new RandomBarrier(100000) }); BarrierBuffer barrierBuffer = new BarrierBuffer(myIG, new BufferSpiller(ioMan, myIG.getPageSize()));
private BufferOrEventSequence rollOver(boolean newBuffer) throws IOException { if (bytesWritten == 0) { return null; } ByteBuffer buf; if (newBuffer) { buf = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); buf.order(ByteOrder.LITTLE_ENDIAN); } else { buf = readBuffer; } // create a reader for the spilled data currentChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize); // create ourselves a new spill file createSpillingChannel(); bytesWritten = 0L; return seq; }
/** * The BufferOrEventSequence returned by this method is safe for concurrent consumption with * any previously returned sequence. * * @return The readable sequence of spilled buffers and events, or 'null', if nothing was added. * @throws IOException Thrown, if the readable sequence could not be created, or no new spill * file could be created. */ @Override public BufferOrEventSequence rollOverWithoutReusingResources() throws IOException { return rollOver(true); }
BarrierBuffer buffer = new BarrierBuffer(gate, new BufferSpiller(ioManager, gate.getPageSize()), 500);
private BufferOrEventSequence rollOver(boolean newBuffer) throws IOException { if (bytesWritten == 0) { return null; } ByteBuffer buf; if (newBuffer) { buf = ByteBuffer.allocateDirect(READ_BUFFER_SIZE); buf.order(ByteOrder.LITTLE_ENDIAN); } else { buf = readBuffer; } // create a reader for the spilled data currentChannel.position(0L); SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(currentSpillFile, currentChannel, buf, pageSize); // create ourselves a new spill file createSpillingChannel(); bytesWritten = 0L; return seq; }