@Override public void triggerCheckpoint(long checkpointId) throws FlinkException { // TODO - we need to see how to derive those. We should probably not encode this in the // TODO - source's trigger message, but do a handshake in this task between the trigger // TODO - message from the master, and the source's trigger notification final CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation(); final long timestamp = System.currentTimeMillis(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp); try { SourceStreamTask.super.triggerCheckpoint(checkpointMetaData, checkpointOptions); } catch (RuntimeException | FlinkException e) { throw e; } catch (Exception e) { throw new FlinkException(e.getMessage(), e); } } };
@Override public Boolean call() throws Exception { for (int i = 0; i < numCheckpoints; i++) { long currentCheckpointId = checkpointId.getAndIncrement(); CheckpointMetaData checkpointMetaData = new CheckpointMetaData(currentCheckpointId, 0L); sourceTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation()); Thread.sleep(checkpointInterval); } return true; } }
private static BufferOrEvent createBarrier(long checkpointId, int channel) { return new BufferOrEvent(new CheckpointBarrier( checkpointId, System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), channel); }
private static BufferOrEvent createBarrier(long id, int channel) { return new BufferOrEvent(new CheckpointBarrier(id, System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), channel); }
@Override protected void run() throws Exception { triggerCheckpointOnBarrier( new CheckpointMetaData( 11L, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetrics()); while (isRunning()) { Thread.sleep(1L); } }
private static BufferOrEvent createBarrier(long id, int channel) { return new BufferOrEvent(new CheckpointBarrier(id, System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), channel); }
/** * Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions, org.apache.flink.runtime.state.CheckpointStreamFactory)}. */ public OperatorSnapshotFinalizer snapshotWithLocalState(long checkpointId, long timestamp) throws Exception { OperatorSnapshotFutures operatorStateResult = operator.snapshotState( checkpointId, timestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), checkpointStorage.resolveCheckpointStorageLocation(checkpointId, CheckpointStorageLocationReference.getDefault())); return new OperatorSnapshotFinalizer(operatorStateResult); }
@Override public Optional<BufferOrEvent> getNextBufferOrEvent() throws IOException, InterruptedException { currentChannel = (currentChannel + 1) % numberOfChannels; if (barrierGens[currentChannel].isNextBarrier()) { return Optional.of( new BufferOrEvent( new CheckpointBarrier( ++currentBarriers[currentChannel], System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), currentChannel)); } else { Buffer buffer = bufferPools[currentChannel].requestBuffer(); buffer.getMemorySegment().putLong(0, c++); return Optional.of(new BufferOrEvent(buffer, currentChannel)); } }
@Override public void run() { try { runStarted.await(); if (getContainingTask().isCanceled() || getContainingTask().triggerCheckpoint( new CheckpointMetaData(0, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation())) { LifecycleTrackingStreamSource.runFinish.trigger(); } } catch (Exception e) { e.printStackTrace(); Assert.fail(); } } };
streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation()); fail("Expected test exception here."); } catch (Exception e) {
checkpointId, timestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(Integer.MAX_VALUE)); fail("Exception expected.");
Whitebox.setInternalState(streamTask, "checkpointStorage", new MemoryBackendCheckpointStorage(new JobID(), null, null, Integer.MAX_VALUE)); streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation()); checkpointCompletedLatch.await(30, TimeUnit.SECONDS); streamTask.cancel();
private void triggerCheckpoint( OneInputStreamTaskTestHarness<String, String> testHarness, OneInputStreamTask<String, String> streamTask) throws Exception { long checkpointId = 1L; CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 1L); testHarness.taskStateManager.setWaitForReportLatch(new OneShotLatch()); while (!streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation())) {} testHarness.taskStateManager.getWaitForReportLatch().await(); long reportedCheckpointId = testHarness.taskStateManager.getReportedCheckpointId(); assertEquals(checkpointId, reportedCheckpointId); }
testHarness.waitForTaskRunning(); testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0); testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0); testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1); testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0); testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1); expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation())); testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1); testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0); testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
/** * Checks that the state snapshot context is closed after a successful snapshot operation. */ @Test public void testSnapshotMethod() throws Exception { final long checkpointId = 42L; final long timestamp = 1L; final CloseableRegistry closeableRegistry = new CloseableRegistry(); StateSnapshotContextSynchronousImpl context = spy(new StateSnapshotContextSynchronousImpl(0L, 0L)); whenNew(StateSnapshotContextSynchronousImpl.class).withAnyArguments().thenReturn(context); StreamTask<Void, AbstractStreamOperator<Void>> containingTask = mock(StreamTask.class); when(containingTask.getCancelables()).thenReturn(closeableRegistry); AbstractStreamOperator<Void> operator = mock(AbstractStreamOperator.class); when(operator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenCallRealMethod(); doReturn(containingTask).when(operator).getContainingTask(); operator.snapshotState( checkpointId, timestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(Integer.MAX_VALUE)); verify(context).close(); }
testHarness.waitForTaskRunning(); testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0); testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1); testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0); testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1); expectedOutput.add(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation())); expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime)); expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
/** * This test checks that tasks emit a proper cancel checkpoint barrier, if a "trigger checkpoint" message * comes before they are ready. */ @Test public void testEmitCancellationBarrierWhenNotReady() throws Exception { StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>( InitBlockingTask::new, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); // start the test - this cannot succeed across the 'init()' method testHarness.invoke(); StreamTask<String, ?> task = testHarness.getTask(); // tell the task to commence a checkpoint boolean result = task.triggerCheckpoint(new CheckpointMetaData(41L, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation()); assertFalse("task triggered checkpoint though not ready", result); // a cancellation barrier should be downstream Object emitted = testHarness.getOutput().poll(); assertNotNull("nothing emitted", emitted); assertTrue("wrong type emitted", emitted instanceof CancelCheckpointMarker); assertEquals("wrong checkpoint id", 41L, ((CancelCheckpointMarker) emitted).getCheckpointId()); }
originalBackend.snapshot(0L, 0L, checkpointStreamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(32, 829), CheckpointOptions.forCheckpointWithDefaultLocation())); assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(34, 900), CheckpointOptions.forCheckpointWithDefaultLocation()));
while (!streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation())) {}