@Override protected void run() throws Exception { // cache processor reference on the stack, to make the code more JIT friendly final StreamTwoInputProcessor<IN1, IN2> inputProcessor = this.inputProcessor; while (running && inputProcessor.processInput()) { // all the work happens in the "processInput" method } }
@Override public void collect(StreamRecord<OUT> record) { if (this.outputTag != null) { // we are only responsible for emitting to the main input return; } pushToRecordWriter(record); }
/** * This method releases all resources of the record writer output. It stops the output * flushing thread (if there is one) and releases all buffers currently held by the output * serializers. * * <p>This method should never fail. */ public void releaseOutputs() { for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.close(); } }
@Override public BarrierBuffer createBarrierHandler(InputGate gate) throws IOException{ return new BarrierBuffer(gate, new BufferSpiller(ioManager, PAGE_SIZE)); }
@Override public BarrierBuffer createBarrierHandler(InputGate gate) throws IOException { return new BarrierBuffer(gate, new CachedBufferBlocker(PAGE_SIZE)); }
private void processEndOfPartition() throws Exception { numClosedChannels++; if (numBarriersReceived > 0) { // let the task know we skip a checkpoint notifyAbort(currentCheckpointId, new InputEndOfStreamException()); // no chance to complete this checkpoint releaseBlocksAndResetBarriers(); } }
/** * Tests that the static HEADER_SIZE field has valid header size. */ @Test public void testHeaderSizeStaticField() throws Exception { int size = 13; BufferOrEvent boe = generateRandomBuffer(size, 0); spiller.add(boe); assertEquals( "Changed the header format, but did not adjust the HEADER_SIZE field", BufferSpiller.HEADER_SIZE + size, spiller.getBytesBlocked()); }
@Test public void testRollOverEmptySequences() throws IOException { BufferBlocker bufferBlocker = createBufferBlocker(); assertNull(bufferBlocker.rollOverReusingResources()); assertNull(bufferBlocker.rollOverReusingResources()); assertNull(bufferBlocker.rollOverReusingResources()); }
/** * The BufferOrEventSequence returned by this method is safe for concurrent consumption with * any previously returned sequence. * * @return The readable sequence of spilled buffers and events, or 'null', if nothing was added. * @throws IOException Thrown, if the readable sequence could not be created, or no new spill * file could be created. */ @Override public BufferOrEventSequence rollOverWithoutReusingResources() throws IOException { return rollOver(true); }
@Override protected void run() throws Exception { // cache processor reference on the stack, to make the code more JIT friendly final StreamInputProcessor<IN> inputProcessor = this.inputProcessor; while (running && inputProcessor.processInput()) { // all the work happens in the "processInput" method } }
@Override protected void cleanup() throws Exception { if (inputProcessor != null) { inputProcessor.cleanup(); } }
@Override protected void cleanup() throws Exception { if (inputProcessor != null) { inputProcessor.cleanup(); } }
@Override public void toggleStreamStatus(StreamStatus status) { if (!status.equals(this.streamStatus)) { this.streamStatus = status; // try and forward the stream status change to all outgoing connections for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.emitStreamStatus(status); } } }
public void broadcastCheckpointBarrier(long id, long timestamp, CheckpointOptions checkpointOptions) throws IOException { CheckpointBarrier barrier = new CheckpointBarrier(id, timestamp, checkpointOptions); for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.broadcastEvent(barrier); } }
/** * It is never reusing resources and is defaulting to {@link #rollOverWithoutReusingResources()}. */ @Override public BufferOrEventSequence rollOverReusingResources() { return rollOverWithoutReusingResources(); }
@Override public BufferOrEventSequence rollOverWithoutReusingResources() { if (bytesBlocked == 0) { return null; } CachedBufferOrEventSequence currentSequence = new CachedBufferOrEventSequence(currentBuffers, bytesBlocked); currentBuffers = new ArrayDeque<BufferOrEvent>(); bytesBlocked = 0L; return currentSequence; }
@Override public Optional<BufferOrEvent> pollNextBufferOrEvent() { return getNextBufferOrEvent(); }
@Override public Optional<BufferOrEvent> pollNextBufferOrEvent() throws IOException, InterruptedException { return getNextBufferOrEvent(); }
/** * NOTE: The BufferOrEventSequences created by this method all reuse the same reading memory * (to reduce overhead) and can consequently not be read concurrently with each other. * * <p>To create a sequence that can be read concurrently with the previous BufferOrEventSequence, * use the {@link #rollOverWithoutReusingResources()} ()} method. * * @return The readable sequence of spilled buffers and events, or 'null', if nothing was added. * @throws IOException Thrown, if the readable sequence could not be created, or no new spill * file could be created. */ @Override public BufferOrEventSequence rollOverReusingResources() throws IOException { return rollOver(false); }