@Override public void triggerCheckpoint(long checkpointId) throws FlinkException { // TODO - we need to see how to derive those. We should probably not encode this in the // TODO - source's trigger message, but do a handshake in this task between the trigger // TODO - message from the master, and the source's trigger notification final CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation(); final long timestamp = System.currentTimeMillis(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp); try { SourceStreamTask.super.triggerCheckpoint(checkpointMetaData, checkpointOptions); } catch (RuntimeException | FlinkException e) { throw e; } catch (Exception e) { throw new FlinkException(e.getMessage(), e); } } };
@Override public void acknowledgeCheckpoint(long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) { taskStateManager.reportTaskStateSnapshots( new CheckpointMetaData(checkpointId, 0L), checkpointMetrics, subtaskState, null); }
@Override public Boolean call() throws Exception { for (int i = 0; i < numCheckpoints; i++) { long currentCheckpointId = checkpointId.getAndIncrement(); CheckpointMetaData checkpointMetaData = new CheckpointMetaData(currentCheckpointId, 0L); sourceTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation()); Thread.sleep(checkpointInterval); } return true; } }
private void notifyCheckpoint(long checkpointId, long timestamp, CheckpointOptions checkpointOptions) throws Exception { if (toNotifyOnCheckpoint != null) { CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp); CheckpointMetrics checkpointMetrics = new CheckpointMetrics() .setBytesBufferedInAlignment(0L) .setAlignmentDurationNanos(0L); toNotifyOnCheckpoint.triggerCheckpointOnBarrier(checkpointMetaData, checkpointOptions, checkpointMetrics); } }
@Override protected void run() throws Exception { triggerCheckpointOnBarrier( new CheckpointMetaData( 11L, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetrics()); while (isRunning()) { Thread.sleep(1L); } }
private void notifyCheckpoint(CheckpointBarrier checkpointBarrier) throws Exception { if (toNotifyOnCheckpoint != null) { CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp()); long bytesBuffered = currentBuffered != null ? currentBuffered.size() : 0L; CheckpointMetrics checkpointMetrics = new CheckpointMetrics() .setBytesBufferedInAlignment(bytesBuffered) .setAlignmentDurationNanos(latestAlignmentDurationNanos); toNotifyOnCheckpoint.triggerCheckpointOnBarrier( checkpointMetaData, checkpointBarrier.getCheckpointOptions(), checkpointMetrics); } }
@Override public void run() { try { runStarted.await(); if (getContainingTask().isCanceled() || getContainingTask().triggerCheckpoint( new CheckpointMetaData(0, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation())) { LifecycleTrackingStreamSource.runFinish.trigger(); } } catch (Exception e) { e.printStackTrace(); Assert.fail(); } } };
CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
@Test public void testRethrowingHandler() { DeclineDummyEnvironment environment = new DeclineDummyEnvironment(); CheckpointExceptionHandlerFactory checkpointExceptionHandlerFactory = new CheckpointExceptionHandlerFactory(); CheckpointExceptionHandler exceptionHandler = checkpointExceptionHandlerFactory.createCheckpointExceptionHandler(true, environment); CheckpointMetaData failedCheckpointMetaData = new CheckpointMetaData(42L, 4711L); Exception testException = new Exception("test"); try { exceptionHandler.tryHandleCheckpointException(failedCheckpointMetaData, testException); Assert.fail("Exception not rethrown."); } catch (Exception e) { Assert.assertEquals(testException, e); } Assert.assertNull(environment.getLastDeclinedCheckpointCause()); }
CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
private void triggerCheckpoint( OneInputStreamTaskTestHarness<String, String> testHarness, OneInputStreamTask<String, String> streamTask) throws Exception { long checkpointId = 1L; CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 1L); testHarness.taskStateManager.setWaitForReportLatch(new OneShotLatch()); while (!streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation())) {} testHarness.taskStateManager.getWaitForReportLatch().await(); long reportedCheckpointId = testHarness.taskStateManager.getReportedCheckpointId(); assertEquals(checkpointId, reportedCheckpointId); }
CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
@Test public void testDecliningHandler() { DeclineDummyEnvironment environment = new DeclineDummyEnvironment(); CheckpointExceptionHandlerFactory checkpointExceptionHandlerFactory = new CheckpointExceptionHandlerFactory(); CheckpointExceptionHandler exceptionHandler = checkpointExceptionHandlerFactory.createCheckpointExceptionHandler(false, environment); CheckpointMetaData failedCheckpointMetaData = new CheckpointMetaData(42L, 4711L); Exception testException = new Exception("test"); try { exceptionHandler.tryHandleCheckpointException(failedCheckpointMetaData, testException); } catch (Exception e) { Assert.fail("Exception not handled, but rethrown."); } Assert.assertEquals(failedCheckpointMetaData.getCheckpointId(), environment.getLastDeclinedCheckpointId()); Assert.assertEquals(testException, environment.getLastDeclinedCheckpointCause()); }
/** * This test checks that tasks emit a proper cancel checkpoint barrier, if a "trigger checkpoint" message * comes before they are ready. */ @Test public void testEmitCancellationBarrierWhenNotReady() throws Exception { StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>( InitBlockingTask::new, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); // start the test - this cannot succeed across the 'init()' method testHarness.invoke(); StreamTask<String, ?> task = testHarness.getTask(); // tell the task to commence a checkpoint boolean result = task.triggerCheckpoint(new CheckpointMetaData(41L, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation()); assertFalse("task triggered checkpoint though not ready", result); // a cancellation barrier should be downstream Object emitted = testHarness.getOutput().poll(); assertNotNull("nothing emitted", emitted); assertTrue("wrong type emitted", emitted instanceof CancelCheckpointMarker); assertEquals("wrong checkpoint id", 41L, ((CancelCheckpointMarker) emitted).getCheckpointId()); }
final AllocationID allocationID = new AllocationID(); final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(42L, 4711L); final CheckpointMetrics checkpointMetrics = new CheckpointMetrics(); final int subtaskIdx = 42;
final long checkpointTimestamp = 1L; final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, checkpointTimestamp); restoredTask.triggerCheckpoint(new CheckpointMetaData(checkpointId, checkpointTimestamp), CheckpointOptions.forCheckpointWithDefaultLocation());
CheckpointMetaData checkpointMetaData = new CheckpointMetaData(0L, 0L); CheckpointMetrics checkpointMetrics = new CheckpointMetrics();
assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(32, 829), CheckpointOptions.forCheckpointWithDefaultLocation())); assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(34, 900), CheckpointOptions.forCheckpointWithDefaultLocation()));
CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, checkpointTimestamp);