private void runTestDeclineOnCheckpointError(AbstractStateBackend backend) throws Exception{ TestDeclinedCheckpointResponder checkpointResponder = new TestDeclinedCheckpointResponder(); Task task = createTask(new FilterOperator(), backend, checkpointResponder, false); // start the task and wait until it is in "restore" task.startTaskThread(); checkpointResponder.declinedLatch.await(); Assert.assertEquals(ExecutionState.RUNNING, task.getExecutionState()); task.cancelExecution(); task.getExecutingThread().join(); }
@Test public void testBlockingNonInterruptibleCheckpoint() throws Exception { StateBackend lockingStateBackend = new BackendForTestStream(LockingOutputStream::new); Task task = createTask(new TestOperator(), lockingStateBackend, mock(CheckpointResponder.class), true); // start the task and wait until it is in "restore" task.startTaskThread(); IN_CHECKPOINT_LATCH.await(); // cancel the task and wait. unless cancellation properly closes // the streams, this will never terminate task.cancelExecution(); task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertNull(task.getFailureCause()); }
@Test public void testCancellationNotBlockedOnLock() throws Exception { syncLatch = new OneShotLatch(); StreamConfig cfg = new StreamConfig(new Configuration()); Task task = createTask(CancelLockingTask.class, cfg, new Configuration()); // start the task and wait until it runs // execution state RUNNING is not enough, we need to wait until the stream task's run() method // is entered task.startTaskThread(); syncLatch.await(); // cancel the execution - this should lead to smooth shutdown task.cancelExecution(); task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
@Test public void testCancellationFailsWithBlockingLock() throws Exception { syncLatch = new OneShotLatch(); StreamConfig cfg = new StreamConfig(new Configuration()); Task task = createTask(CancelFailingTask.class, cfg, new Configuration()); // start the task and wait until it runs // execution state RUNNING is not enough, we need to wait until the stream task's run() method // is entered task.startTaskThread(); syncLatch.await(); // cancel the execution - this should lead to smooth shutdown task.cancelExecution(); task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
/** * This test checks that cancel calls that are issued before the operator is * instantiated still lead to proper canceling. */ @Test public void testEarlyCanceling() throws Exception { final StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setOperatorID(new OperatorID(4711L, 42L)); cfg.setStreamOperator(new SlowlyDeserializingOperator()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions()); final Task task = createTask(SourceStreamTask.class, cfg, new Configuration(), taskManagerActions); final TaskExecutionState state = new TaskExecutionState( task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING); task.startTaskThread(); verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state)); // send a cancel. because the operator takes a long time to deserialize, this should // hit the task before the operator is deserialized task.cancelExecution(); task.getExecutingThread().join(); assertFalse("Task did not cancel", task.getExecutingThread().isAlive()); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
@Test public void testLifeCycleCancel() throws Exception { ACTUAL_ORDER_TRACKING.clear(); Configuration taskManagerConfig = new Configuration(); StreamConfig cfg = new StreamConfig(new Configuration()); MockSourceFunction srcFun = new MockSourceFunction(); cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false)); cfg.setOperatorID(new OperatorID()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig); task.startTaskThread(); LifecycleTrackingStreamSource.runStarted.await(); // this should cancel the task even though it is blocked on runFinished task.cancelExecution(); // wait for clean termination task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING); }
private void testRestoreWithInterrupt(int mode) throws Exception { IN_RESTORE_LATCH.reset(); Configuration taskConfig = new Configuration(); StreamConfig cfg = new StreamConfig(taskConfig); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); switch (mode) { case OPERATOR_MANAGED: case OPERATOR_RAW: case KEYED_MANAGED: case KEYED_RAW: cfg.setStateKeySerializer(IntSerializer.INSTANCE); cfg.setStreamOperator(new StreamSource<>(new TestSource(mode))); break; default: throw new IllegalArgumentException(); } StreamStateHandle lockingHandle = new InterruptLockingStateHandle(); Task task = createTask(cfg, taskConfig, lockingHandle, mode); // start the task and wait until it is in "restore" task.startTaskThread(); IN_RESTORE_LATCH.await(); // trigger cancellation and signal to continue task.cancelExecution(); task.getExecutingThread().join(30000); if (task.getExecutionState() == ExecutionState.CANCELING) { fail("Task is stuck and not canceling"); } assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertNull(task.getFailureCause()); }
@Override public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) { final Task task = taskSlotTable.getTask(executionAttemptID); if (task != null) { try { task.cancelExecution(); return CompletableFuture.completedFuture(Acknowledge.get()); } catch (Throwable t) { return FutureUtils.completedExceptionally( new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t)); } } else { final String message = "Cannot find task to stop for execution " + executionAttemptID + '.'; log.debug(message); return FutureUtils.completedExceptionally(new TaskException(message)); } }
@Override public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) { final Task task = taskSlotTable.getTask(executionAttemptID); if (task != null) { try { task.cancelExecution(); return CompletableFuture.completedFuture(Acknowledge.get()); } catch (Throwable t) { return FutureUtils.completedExceptionally( new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t)); } } else { final String message = "Cannot find task to stop for execution " + executionAttemptID + '.'; log.debug(message); return FutureUtils.completedExceptionally(new TaskException(message)); } }
@Override public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) { final Task task = taskSlotTable.getTask(executionAttemptID); if (task != null) { try { task.cancelExecution(); return CompletableFuture.completedFuture(Acknowledge.get()); } catch (Throwable t) { return FutureUtils.completedExceptionally( new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t)); } } else { final String message = "Cannot find task to stop for execution " + executionAttemptID + '.'; log.debug(message); return FutureUtils.completedExceptionally(new TaskException(message)); } }
@RpcMethod public Acknowledge cancelTask(ExecutionAttemptID executionAttemptID) throws TaskException { final Task task = taskSlotTable.getTask(executionAttemptID); if (task != null) { try { task.cancelExecution(); return Acknowledge.get(); } catch (Throwable t) { throw new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t); } } else { final String message = "Cannot find task to stop for execution " + executionAttemptID + '.'; log.debug(message); throw new TaskException(message); } }
@Override public Void apply(ExecutionState executionState, Throwable throwable) { try { if (executionState != null) { onPartitionStateUpdate( intermediateDataSetId, resultPartitionId, executionState); } else if (throwable instanceof TimeoutException) { // our request timed out, assume we're still running and try again onPartitionStateUpdate( intermediateDataSetId, resultPartitionId, ExecutionState.RUNNING); } else if (throwable instanceof PartitionProducerDisposedException) { String msg = String.format("Producer %s of partition %s disposed. Cancelling execution.", resultPartitionId.getProducerId(), resultPartitionId.getPartitionId()); LOG.info(msg, throwable); cancelExecution(); } else { failExternally(throwable); } } catch (IOException | InterruptedException e) { failExternally(e); } return null; } }, executor);
resultPartitionId.getProducerId(), resultPartitionId.getPartitionId()); LOG.info(msg, throwable); cancelExecution(); } else { failExternally(throwable);
resultPartitionId.getProducerId(), resultPartitionId.getPartitionId()); LOG.info(msg, throwable); cancelExecution(); } else { failExternally(throwable);
resultPartitionId.getProducerId(), resultPartitionId.getPartitionId()); LOG.info(msg, throwable); cancelExecution(); } else { failExternally(throwable);