private Throwable runTestTaskFailingOnCheckpointError(AbstractStateBackend backend) throws Exception { Task task = createTask(new FilterOperator(), backend, mock(CheckpointResponder.class), true); // start the task and wait until it is in "restore" task.startTaskThread(); task.getExecutingThread().join(); assertEquals(ExecutionState.FAILED, task.getExecutionState()); return task.getFailureCause(); }
private void runTestDeclineOnCheckpointError(AbstractStateBackend backend) throws Exception{ TestDeclinedCheckpointResponder checkpointResponder = new TestDeclinedCheckpointResponder(); Task task = createTask(new FilterOperator(), backend, checkpointResponder, false); // start the task and wait until it is in "restore" task.startTaskThread(); checkpointResponder.declinedLatch.await(); Assert.assertEquals(ExecutionState.RUNNING, task.getExecutionState()); task.cancelExecution(); task.getExecutingThread().join(); }
@Test public void testBlockingNonInterruptibleCheckpoint() throws Exception { StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new); Task task = createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class), true); // start the task and wait until it is in "restore" task.startTaskThread(); IN_CHECKPOINT_LATCH.await(); // cancel the task and wait. unless cancellation properly closes // the streams, this will never terminate task.cancelExecution(); task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertNull(task.getFailureCause()); }
@Test public void testStateBackendLoadingAndClosing() throws Exception { Configuration taskManagerConfig = new Configuration(); taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName()); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateKeySerializer(mock(TypeSerializer.class)); cfg.setOperatorID(new OperatorID(4711L, 42L)); TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction()); cfg.setStreamOperator(streamSource); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig); StateBackendTestSource.fail = false; task.startTaskThread(); // wait for clean termination task.getExecutingThread().join(); // ensure that the state backends and stream iterables are closed ... verify(TestStreamSource.operatorStateBackend).close(); verify(TestStreamSource.keyedStateBackend).close(); verify(TestStreamSource.rawOperatorStateInputs).close(); verify(TestStreamSource.rawKeyedStateInputs).close(); // ... and disposed verify(TestStreamSource.operatorStateBackend).dispose(); verify(TestStreamSource.keyedStateBackend).dispose(); assertEquals(ExecutionState.FINISHED, task.getExecutionState()); }
@Test public void testStateBackendClosingOnFailure() throws Exception { Configuration taskManagerConfig = new Configuration(); taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName()); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateKeySerializer(mock(TypeSerializer.class)); cfg.setOperatorID(new OperatorID(4711L, 42L)); TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction()); cfg.setStreamOperator(streamSource); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig); StateBackendTestSource.fail = true; task.startTaskThread(); // wait for clean termination task.getExecutingThread().join(); // ensure that the state backends and stream iterables are closed ... verify(TestStreamSource.operatorStateBackend).close(); verify(TestStreamSource.keyedStateBackend).close(); verify(TestStreamSource.rawOperatorStateInputs).close(); verify(TestStreamSource.rawKeyedStateInputs).close(); // ... and disposed verify(TestStreamSource.operatorStateBackend).dispose(); verify(TestStreamSource.keyedStateBackend).dispose(); assertEquals(ExecutionState.FAILED, task.getExecutionState()); }
/** * This test checks that cancel calls that are issued before the operator is * instantiated still lead to proper canceling. */ @Test public void testEarlyCanceling() throws Exception { final StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setOperatorID(new OperatorID(4711L, 42L)); cfg.setStreamOperator(new SlowlyDeserializingOperator()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions()); final Task task = createTask(SourceStreamTask.class, cfg, new Configuration(), taskManagerActions); final TaskExecutionState state = new TaskExecutionState( task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING); task.startTaskThread(); verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state)); // send a cancel. because the operator takes a long time to deserialize, this should // hit the task before the operator is deserialized task.cancelExecution(); task.getExecutingThread().join(); assertFalse("Task did not cancel", task.getExecutingThread().isAlive()); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
@Test public void testCancellationNotBlockedOnLock() throws Exception { syncLatch = new OneShotLatch(); StreamConfig cfg = new StreamConfig(new Configuration()); Task task = createTask(CancelLockingTask.class, cfg, new Configuration()); // start the task and wait until it runs // execution state RUNNING is not enough, we need to wait until the stream task's run() method // is entered task.startTaskThread(); syncLatch.await(); // cancel the execution - this should lead to smooth shutdown task.cancelExecution(); task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
@Test public void testCancellationFailsWithBlockingLock() throws Exception { syncLatch = new OneShotLatch(); StreamConfig cfg = new StreamConfig(new Configuration()); Task task = createTask(CancelFailingTask.class, cfg, new Configuration()); // start the task and wait until it runs // execution state RUNNING is not enough, we need to wait until the stream task's run() method // is entered task.startTaskThread(); syncLatch.await(); // cancel the execution - this should lead to smooth shutdown task.cancelExecution(); task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
@Test public void testLifeCycleFull() throws Exception { ACTUAL_ORDER_TRACKING.clear(); Configuration taskManagerConfig = new Configuration(); StreamConfig cfg = new StreamConfig(new Configuration()); MockSourceFunction srcFun = new MockSourceFunction(); cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true)); cfg.setOperatorID(new OperatorID()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig); task.startTaskThread(); LifecycleTrackingStreamSource.runStarted.await(); // wait for clean termination task.getExecutingThread().join(); assertEquals(ExecutionState.FINISHED, task.getExecutionState()); assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING); }
@Test public void testLifeCycleCancel() throws Exception { ACTUAL_ORDER_TRACKING.clear(); Configuration taskManagerConfig = new Configuration(); StreamConfig cfg = new StreamConfig(new Configuration()); MockSourceFunction srcFun = new MockSourceFunction(); cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false)); cfg.setOperatorID(new OperatorID()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig); task.startTaskThread(); LifecycleTrackingStreamSource.runStarted.await(); // this should cancel the task even though it is blocked on runFinished task.cancelExecution(); // wait for clean termination task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING); }
private void testRestoreWithInterrupt(int mode) throws Exception { IN_RESTORE_LATCH.reset(); Configuration taskConfig = new Configuration(); StreamConfig cfg = new StreamConfig(taskConfig); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); switch (mode) { case OPERATOR_MANAGED: case OPERATOR_RAW: case KEYED_MANAGED: case KEYED_RAW: cfg.setStateKeySerializer(IntSerializer.INSTANCE); cfg.setStreamOperator(new StreamSource<>(new TestSource(mode))); break; default: throw new IllegalArgumentException(); } StreamStateHandle lockingHandle = new InterruptLockingStateHandle(); Task task = createTask(cfg, taskConfig, lockingHandle, mode); // start the task and wait until it is in "restore" task.startTaskThread(); IN_RESTORE_LATCH.await(); // trigger cancellation and signal to continue task.cancelExecution(); task.getExecutingThread().join(30000); if (task.getExecutionState() == ExecutionState.CANCELING) { fail("Task is stuck and not canceling"); } assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertNull(task.getFailureCause()); }
private Optional<StackTraceElement[]> getStackTrace( final ExecutionAttemptID executionAttemptId, final int maxStackTraceDepth) { final Task task = taskSlotTable.getTask(executionAttemptId); if (task != null && task.getExecutionState() == ExecutionState.RUNNING) { final StackTraceElement[] stackTrace = task.getExecutingThread().getStackTrace(); if (maxStackTraceDepth > 0) { return Optional.of(Arrays.copyOfRange(stackTrace, 0, Math.min(maxStackTraceDepth, stackTrace.length))); } else { return Optional.of(stackTrace); } } else { return Optional.empty(); } }
private Optional<StackTraceElement[]> getStackTrace( final ExecutionAttemptID executionAttemptId, final int maxStackTraceDepth) { final Task task = taskSlotTable.getTask(executionAttemptId); if (task != null && task.getExecutionState() == ExecutionState.RUNNING) { final StackTraceElement[] stackTrace = task.getExecutingThread().getStackTrace(); if (maxStackTraceDepth > 0) { return Optional.of(Arrays.copyOfRange(stackTrace, 0, Math.min(maxStackTraceDepth, stackTrace.length))); } else { return Optional.of(stackTrace); } } else { return Optional.empty(); } }
private Optional<StackTraceElement[]> getStackTrace( final ExecutionAttemptID executionAttemptId, final int maxStackTraceDepth) { final Task task = taskSlotTable.getTask(executionAttemptId); if (task != null && task.getExecutionState() == ExecutionState.RUNNING) { final StackTraceElement[] stackTrace = task.getExecutingThread().getStackTrace(); if (maxStackTraceDepth > 0) { return Optional.of(Arrays.copyOfRange(stackTrace, 0, Math.min(maxStackTraceDepth, stackTrace.length))); } else { return Optional.of(stackTrace); } } else { return Optional.empty(); } }