private void notifyAbort(long checkpointId, CheckpointDeclineException cause) throws Exception { if (toNotifyOnCheckpoint != null) { toNotifyOnCheckpoint.abortCheckpointOnBarrier(checkpointId, cause); } }
@Override public boolean hasNext() { if (exhausted) { return false; } if (nextSplit != null) { return true; } final InputSplit split; try { split = provider.getNextInputSplit(getRuntimeContext().getUserCodeClassLoader()); } catch (InputSplitProviderException e) { throw new RuntimeException("Could not retrieve next input split.", e); } if (split != null) { this.nextSplit = split; return true; } else { exhausted = true; return false; } }
private void notifyCheckpoint(long checkpointId, long timestamp, CheckpointOptions checkpointOptions) throws Exception { if (toNotifyOnCheckpoint != null) { CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp); CheckpointMetrics checkpointMetrics = new CheckpointMetrics() .setBytesBufferedInAlignment(0L) .setAlignmentDurationNanos(0L); toNotifyOnCheckpoint.triggerCheckpointOnBarrier(checkpointMetaData, checkpointOptions, checkpointMetrics); } }
verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); assertEquals(0L, buffer.getAlignmentDurationNanos()); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(2L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(4L), any(CheckpointDeclineOnCancellationBarrierException.class)); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(5L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); assertEquals(0L, buffer.getAlignmentDurationNanos()); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(6L), any(CheckpointDeclineOnCancellationBarrierException.class)); assertEquals(0L, buffer.getAlignmentDurationNanos());
/** * Tests that disabled checkpointing sets the checkpointing interval to Long.MAX_VALUE. */ @Test public void testDisabledCheckpointing() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); StreamGraph streamGraph = new StreamGraph(env); assertFalse("Checkpointing enabled", streamGraph.getCheckpointConfig().isCheckpointingEnabled()); JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph); JobCheckpointingSettings snapshottingSettings = jobGraph.getCheckpointingSettings(); assertEquals(Long.MAX_VALUE, snapshottingSettings.getCheckpointCoordinatorConfiguration().getCheckpointInterval()); }
JobCheckpointingSettings settings = new JobCheckpointingSettings( triggerVertices, ackVertices, commitVertices, new CheckpointCoordinatorConfiguration( interval, cfg.getCheckpointTimeout(),
SerializedValue<Factory[]> serializedConfiguredHooks = jg.getCheckpointingSettings().getMasterHooks(); assertNotNull(serializedConfiguredHooks);
verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(3L), any(CheckpointDeclineSubsumedException.class)); check(sequence[4], buffer.getNextNonBlocked(), PAGE_SIZE); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(5L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); check(sequence[11], buffer.getNextNonBlocked(), PAGE_SIZE); verify(toNotify, times(1)).triggerCheckpointOnBarrier(any(CheckpointMetaData.class), any(CheckpointOptions.class), any(CheckpointMetrics.class)); verify(toNotify, times(1)).abortCheckpointOnBarrier(anyLong(), any(Throwable.class));
/** * Checks if the checkpointing was enabled for this job graph * * @return true if checkpointing enabled */ public boolean isCheckpointingEnabled() { if (snapshotSettings == null) { return false; } long checkpointInterval = snapshotSettings.getCheckpointCoordinatorConfiguration().getCheckpointInterval(); return checkpointInterval > 0 && checkpointInterval < Long.MAX_VALUE; }
private void notifyAbort(long checkpointId) throws Exception { if (toNotifyOnCheckpoint != null) { toNotifyOnCheckpoint.abortCheckpointOnBarrier( checkpointId, new CheckpointDeclineOnCancellationBarrierException()); } }
private void notifyCheckpoint(CheckpointBarrier checkpointBarrier) throws Exception { if (toNotifyOnCheckpoint != null) { CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp()); long bytesBuffered = currentBuffered != null ? currentBuffered.size() : 0L; CheckpointMetrics checkpointMetrics = new CheckpointMetrics() .setBytesBufferedInAlignment(bytesBuffered) .setAlignmentDurationNanos(latestAlignmentDurationNanos); toNotifyOnCheckpoint.triggerCheckpointOnBarrier( checkpointMetaData, checkpointBarrier.getCheckpointOptions(), checkpointMetrics); } }
verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); validateAlignmentTime(startTs, buffer.getAlignmentDurationNanos()); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(2L), any(CheckpointDeclineOnCancellationBarrierException.class)); assertEquals(0L, buffer.getAlignmentDurationNanos()); verify(toNotify, times(1)).triggerCheckpointOnBarrier(any(CheckpointMetaData.class), any(CheckpointOptions.class), any(CheckpointMetrics.class)); verify(toNotify, times(1)).abortCheckpointOnBarrier(anyLong(), any(CheckpointDeclineOnCancellationBarrierException.class));
/** * Tests that each checkpoint is only aborted once in case of an interleaved cancellation * barrier arrival of two consecutive checkpoints. */ @Test public void testInterleavedCancellationBarriers() throws Exception { BufferOrEvent[] sequence = { createBarrier(1L, 0), createCancellationBarrier(2L, 0), createCancellationBarrier(1L, 1), createCancellationBarrier(2L, 1), createCancellationBarrier(1L, 2), createCancellationBarrier(2L, 2), createBuffer(0) }; MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence)); BarrierTracker tracker = new BarrierTracker(gate); AbstractInvokable statefulTask = mock(AbstractInvokable.class); tracker.registerCheckpointEventHandler(statefulTask); for (BufferOrEvent boe : sequence) { if (boe.isBuffer() || (boe.getEvent().getClass() != CheckpointBarrier.class && boe.getEvent().getClass() != CancelCheckpointMarker.class)) { assertEquals(boe, tracker.getNextNonBlocked()); } } verify(statefulTask, times(1)).abortCheckpointOnBarrier(eq(1L), any(Throwable.class)); verify(statefulTask, times(1)).abortCheckpointOnBarrier(eq(2L), any(Throwable.class)); }
verify(toNotify).abortCheckpointOnBarrier(eq(1L), any(CheckpointDeclineOnCancellationBarrierException.class)); verify(toNotify).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(2L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); verify(toNotify, times(1)).triggerCheckpointOnBarrier(any(CheckpointMetaData.class), any(CheckpointOptions.class), any(CheckpointMetrics.class)); verify(toNotify, times(1)).abortCheckpointOnBarrier(anyLong(), any(Throwable.class));
private void notifyAbort(long checkpointId, CheckpointDeclineException cause) throws Exception { if (toNotifyOnCheckpoint != null) { toNotifyOnCheckpoint.abortCheckpointOnBarrier(checkpointId, cause); } }
startTs = System.nanoTime(); check(sequence[5], buffer.getNextNonBlocked(), PAGE_SIZE); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); validateAlignmentTime(startTs, buffer.getAlignmentDurationNanos()); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(2L), any(CheckpointDeclineOnCancellationBarrierException.class)); validateAlignmentTime(startTs, buffer.getAlignmentDurationNanos()); check(sequence[13], buffer.getNextNonBlocked(), PAGE_SIZE); startTs = System.nanoTime(); check(sequence[20], buffer.getNextNonBlocked(), PAGE_SIZE); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(3L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); validateAlignmentTime(startTs, buffer.getAlignmentDurationNanos()); check(sequence[21], buffer.getNextNonBlocked(), PAGE_SIZE); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(4L), any(CheckpointDeclineOnCancellationBarrierException.class)); assertEquals(0L, buffer.getAlignmentDurationNanos()); verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(5L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); validateAlignmentTime(startTs, buffer.getAlignmentDurationNanos()); check(sequence[33], buffer.getNextNonBlocked(), PAGE_SIZE); verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(6L), any(CheckpointDeclineOnCancellationBarrierException.class)); assertEquals(0L, buffer.getAlignmentDurationNanos());
private void notifyAbort(long checkpointId, CheckpointDeclineException cause) throws Exception { if (toNotifyOnCheckpoint != null) { toNotifyOnCheckpoint.abortCheckpointOnBarrier(checkpointId, cause); } }
verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(3L), any(AlignmentLimitExceededException.class)); verify(toNotify, times(1)).triggerCheckpointOnBarrier( argThat(new CheckpointMatcher(4L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); verify(toNotify, times(0)).triggerCheckpointOnBarrier( argThat(new CheckpointMatcher(3L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(7L), any(AlignmentLimitExceededException.class)); verify(toNotify, times(0)).triggerCheckpointOnBarrier( any(CheckpointMetaData.class), any(CheckpointOptions.class),
verify(toNotify).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class)); check(sequence[6], buffer.getNextNonBlocked(), PAGE_SIZE); assertEquals(3L, buffer.getCurrentCheckpointId()); validateAlignmentTime(startTs, buffer.getAlignmentDurationNanos()); verify(toNotify).abortCheckpointOnBarrier(eq(2L), isA(CheckpointDeclineSubsumedException.class)); check(sequence[16], buffer.getNextNonBlocked(), PAGE_SIZE); verify(toNotify).abortCheckpointOnBarrier(eq(3L), isA(InputEndOfStreamException.class));