@Override public BarrierBuffer createBarrierHandler(InputGate gate) throws IOException{ return new BarrierBuffer(gate, new BufferSpiller(ioManager, PAGE_SIZE)); }
next = Optional.ofNullable(currentBuffered.getNext()); if (!next.isPresent()) { completeBufferedSequence(); return getNextNonBlocked(); releaseBlocksAndResetBarriers(); return getNextNonBlocked(); if (isBlocked(bufferOrEvent.getChannelIndex())) { checkSizeLimit(); if (!endOfStream) { processBarrier((CheckpointBarrier) bufferOrEvent.getEvent(), bufferOrEvent.getChannelIndex()); processCancellationBarrier((CancelCheckpointMarker) bufferOrEvent.getEvent()); processEndOfPartition();
notifyCheckpoint(receivedBarrier); onBarrier(channelIndex); notifyAbort(currentCheckpointId, new CheckpointDeclineSubsumedException(barrierId)); releaseBlocksAndResetBarriers(); beginNewAlignment(barrierId, channelIndex); beginNewAlignment(barrierId, channelIndex); releaseBlocksAndResetBarriers(); notifyCheckpoint(receivedBarrier);
private void processEndOfPartition() throws Exception { numClosedChannels++; if (numBarriersReceived > 0) { // let the task know we skip a checkpoint notifyAbort(currentCheckpointId, new InputEndOfStreamException()); // no chance to complete this checkpoint releaseBlocksAndResetBarriers(); } }
notifyAbortOnCancellationBarrier(barrierId); releaseBlocksAndResetBarriers(); notifyAbortOnCancellationBarrier(barrierId); releaseBlocksAndResetBarriers(); latestAlignmentDurationNanos = 0L; notifyAbort(currentCheckpointId, new CheckpointDeclineSubsumedException(barrierId)); notifyAbortOnCancellationBarrier(barrierId); notifyAbortOnCancellationBarrier(barrierId);
buffer.registerCheckpointEventHandler(toNotify); check(sequence[0], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[2], buffer.getNextNonBlocked(), PAGE_SIZE); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); assertEquals(0L, buffer.getAlignmentDurationNanos()); check(sequence[6], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(5L, buffer.getCurrentCheckpointId()); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(2L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(4L), any(CheckpointDeclineOnCancellationBarrierException.class)); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(5L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); assertEquals(0L, buffer.getAlignmentDurationNanos()); check(sequence[8], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(6L, buffer.getCurrentCheckpointId()); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(6L), any(CheckpointDeclineOnCancellationBarrierException.class)); assertEquals(0L, buffer.getAlignmentDurationNanos()); buffer.cleanup();
BarrierBuffer buffer = new BarrierBuffer(gate, new BufferSpiller(ioManager, gate.getPageSize()), 1000); buffer.registerCheckpointEventHandler(toNotify); check(sequence[0], buffer.getNextNonBlocked()); check(sequence[1], buffer.getNextNonBlocked()); check(sequence[2], buffer.getNextNonBlocked()); check(sequence[3], buffer.getNextNonBlocked()); check(sequence[6], buffer.getNextNonBlocked()); check(sequence[8], buffer.getNextNonBlocked()); check(sequence[10], buffer.getNextNonBlocked()); check(sequence[5], buffer.getNextNonBlocked()); validateAlignmentTime(startTs, buffer); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(7L), any(AlignmentLimitExceededException.class)); check(sequence[7], buffer.getNextNonBlocked()); check(sequence[11], buffer.getNextNonBlocked()); check(sequence[12], buffer.getNextNonBlocked()); check(sequence[13], buffer.getNextNonBlocked()); check(sequence[14], buffer.getNextNonBlocked()); check(sequence[15], buffer.getNextNonBlocked()); check(sequence[16], buffer.getNextNonBlocked()); check(sequence[17], buffer.getNextNonBlocked()); check(sequence[19], buffer.getNextNonBlocked()); check(sequence[20], buffer.getNextNonBlocked());
new BarrierGenerator[] { new CountBarrier(100000), new RandomBarrier(100000) }); BarrierBuffer barrierBuffer = new BarrierBuffer(myIG, new BufferSpiller(ioMan, myIG.getPageSize())); BufferOrEvent boe = barrierBuffer.getNextNonBlocked(); if (boe.isBuffer()) { boe.getBuffer().recycleBuffer();
check(sequence[3], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[4], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[5], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(1L, buffer.getCurrentCheckpointId()); check(sequence[10], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(2L, buffer.getCurrentCheckpointId()); check(sequence[12], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[8], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[9], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[11], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[13], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[14], buffer.getNextNonBlocked(), PAGE_SIZE); assertNull(buffer.getNextNonBlocked()); assertNull(buffer.getNextNonBlocked()); buffer.cleanup();
/** * Validates that the buffer behaves correctly if no checkpoint barriers come, * for a single input channel. */ @Test public void testSingleChannelNoBarriers() throws Exception { BufferOrEvent[] sequence = { createBuffer(0, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createEndOfPartition(0) }; MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence)); BarrierBuffer buffer = createBarrierHandler(gate); for (BufferOrEvent boe : sequence) { assertEquals(boe, buffer.getNextNonBlocked()); } assertEquals(0L, buffer.getAlignmentDurationNanos()); assertNull(buffer.getNextNonBlocked()); assertNull(buffer.getNextNonBlocked()); buffer.cleanup(); }
buffer.registerCheckpointEventHandler(handler); handler.setNextExpectedCheckpointId(1L); check(sequence[0], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[1], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[2], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(1L, handler.getNextExpectedCheckpointId()); check(sequence[6], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(2L, handler.getNextExpectedCheckpointId()); check(sequence[7], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[8], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[13], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[14], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[18], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[19], buffer.getNextNonBlocked(), PAGE_SIZE); buffer.getNextNonBlocked(); buffer.cleanup();
private void beginNewAlignment(long checkpointId, int channelIndex) throws IOException { currentCheckpointId = checkpointId; onBarrier(channelIndex); startOfAlignmentTimestamp = System.nanoTime(); if (LOG.isDebugEnabled()) { LOG.debug("{}: Starting stream alignment for checkpoint {}.", inputGate.getOwningTaskName(), checkpointId); } }
private void notifyAbortOnCancellationBarrier(long checkpointId) throws Exception { notifyAbort(checkpointId, new CheckpointDeclineOnCancellationBarrierException()); }
buffer.registerCheckpointEventHandler(toNotify); check(sequence[0], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[2], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[3], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[6], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[4], buffer.getNextNonBlocked(), PAGE_SIZE); validateAlignmentTime(startTs, buffer.getAlignmentDurationNanos()); verify(toNotify).abortCheckpointOnBarrier(eq(1L), any(CheckpointDeclineOnCancellationBarrierException.class)); check(sequence[9], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[11], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[13], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[15], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[7], buffer.getNextNonBlocked(), PAGE_SIZE); validateAlignmentTime(startTs, buffer.getAlignmentDurationNanos()); verify(toNotify).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(2L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); check(sequence[10], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[14], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[18], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[19], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[20], buffer.getNextNonBlocked(), PAGE_SIZE); assertNull(buffer.getNextNonBlocked());
notifyAbortOnCancellationBarrier(barrierId); releaseBlocksAndResetBarriers(); notifyAbortOnCancellationBarrier(barrierId); releaseBlocksAndResetBarriers(); latestAlignmentDurationNanos = 0L; notifyAbort(currentCheckpointId, new CheckpointDeclineSubsumedException(barrierId)); notifyAbortOnCancellationBarrier(barrierId); notifyAbortOnCancellationBarrier(barrierId);
private void checkSizeLimit() throws Exception { if (maxBufferedBytes > 0 && (numQueuedBytes + bufferBlocker.getBytesBlocked()) > maxBufferedBytes) { // exceeded our limit - abort this checkpoint LOG.info("{}: Checkpoint {} aborted because alignment volume limit ({} bytes) exceeded.", inputGate.getOwningTaskName(), currentCheckpointId, maxBufferedBytes); releaseBlocksAndResetBarriers(); notifyAbort(currentCheckpointId, new AlignmentLimitExceededException(maxBufferedBytes)); } }
BarrierBuffer buffer = new BarrierBuffer(gate, new BufferSpiller(ioManager, gate.getPageSize()), 500); buffer.registerCheckpointEventHandler(toNotify); check(sequence[0], buffer.getNextNonBlocked()); check(sequence[1], buffer.getNextNonBlocked()); check(sequence[3], buffer.getNextNonBlocked()); check(sequence[7], buffer.getNextNonBlocked()); check(sequence[11], buffer.getNextNonBlocked()); check(sequence[4], buffer.getNextNonBlocked()); validateAlignmentTime(startTs, buffer); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(3L), any(AlignmentLimitExceededException.class)); check(sequence[6], buffer.getNextNonBlocked()); startTs = System.nanoTime(); check(sequence[12], buffer.getNextNonBlocked()); check(sequence[17], buffer.getNextNonBlocked()); check(sequence[9], buffer.getNextNonBlocked()); validateAlignmentTime(startTs, buffer); verify(toNotify, times(1)).triggerCheckpointOnBarrier( argThat(new CheckpointMatcher(4L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); check(sequence[10], buffer.getNextNonBlocked()); check(sequence[15], buffer.getNextNonBlocked());
check(sequence[0], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[1], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[2], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[3], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[4], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[7], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(2L, buffer.getCurrentCheckpointId()); check(sequence[8], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[11], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[10], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(3L, buffer.getCurrentCheckpointId()); check(sequence[15], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(4L, buffer.getCurrentCheckpointId()); check(sequence[16], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[17], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[18], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[19], buffer.getNextNonBlocked(), PAGE_SIZE); check(sequence[21], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(5L, buffer.getCurrentCheckpointId()); check(sequence[22], buffer.getNextNonBlocked(), PAGE_SIZE); assertNull(buffer.getNextNonBlocked()); assertNull(buffer.getNextNonBlocked()); buffer.cleanup();
/** * Validates that the buffer behaves correctly if no checkpoint barriers come, * for an input with multiple input channels. */ @Test public void testMultiChannelNoBarriers() throws Exception { BufferOrEvent[] sequence = { createBuffer(2, PAGE_SIZE), createBuffer(2, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createBuffer(1, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createEndOfPartition(0), createBuffer(3, PAGE_SIZE), createBuffer(1, PAGE_SIZE), createEndOfPartition(3), createBuffer(1, PAGE_SIZE), createEndOfPartition(1), createBuffer(2, PAGE_SIZE), createEndOfPartition(2) }; MockInputGate gate = new MockInputGate(PAGE_SIZE, 4, Arrays.asList(sequence)); BarrierBuffer buffer = createBarrierHandler(gate); for (BufferOrEvent boe : sequence) { assertEquals(boe, buffer.getNextNonBlocked()); } assertEquals(0L, buffer.getAlignmentDurationNanos()); assertNull(buffer.getNextNonBlocked()); assertNull(buffer.getNextNonBlocked()); buffer.cleanup(); }
/** * Validates that the buffer preserved the order of elements for a * input with a single input channel, and checkpoint events. */ @Test public void testSingleChannelWithBarriers() throws Exception { BufferOrEvent[] sequence = { createBuffer(0, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createBarrier(1, 0), createBuffer(0, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createBarrier(2, 0), createBarrier(3, 0), createBuffer(0, PAGE_SIZE), createBuffer(0, PAGE_SIZE), createBarrier(4, 0), createBarrier(5, 0), createBarrier(6, 0), createBuffer(0, PAGE_SIZE), createEndOfPartition(0) }; MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence)); BarrierBuffer buffer = createBarrierHandler(gate); ValidatingCheckpointHandler handler = new ValidatingCheckpointHandler(); buffer.registerCheckpointEventHandler(handler); handler.setNextExpectedCheckpointId(1L); for (BufferOrEvent boe : sequence) { if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) { assertEquals(boe, buffer.getNextNonBlocked()); } } assertNull(buffer.getNextNonBlocked()); assertNull(buffer.getNextNonBlocked()); buffer.cleanup(); }