private void connect(Integer headOfChain, StreamEdge edge) { physicalEdgesInOrder.add(edge); Integer downStreamvertexID = edge.getTargetId(); JobVertex headVertex = jobVertices.get(headOfChain); JobVertex downStreamVertex = jobVertices.get(downStreamvertexID); StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration()); downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1); StreamPartitioner<?> partitioner = edge.getPartitioner(); JobEdge jobEdge; if (partitioner instanceof ForwardPartitioner || partitioner instanceof RescalePartitioner) { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED_BOUNDED); } else { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED_BOUNDED); } // set strategy name so that web interface can show it. jobEdge.setShipStrategyName(partitioner.toString()); if (LOG.isDebugEnabled()) { LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(), headOfChain, downStreamvertexID); } }
/** * Constructor for initialization, possibly with initial state (recovery / savepoint / etc). * * <p>This constructor accepts a special {@link ProcessingTimeService}. By default (and if * null is passes for the time provider) a {@link SystemProcessingTimeService DefaultTimerService} * will be used. * * @param environment The task environment for this task. * @param timeProvider Optionally, a specific time provider to use. */ protected StreamTask( Environment environment, @Nullable ProcessingTimeService timeProvider) { super(environment); this.timerService = timeProvider; this.configuration = new StreamConfig(getTaskConfiguration()); this.accumulatorMap = getEnvironment().getAccumulatorRegistry().getUserMap(); this.recordWriters = createRecordWriters(configuration, environment); }
public StreamingRuntimeContext(AbstractStreamOperator<?> operator, Environment env, Map<String, Accumulator<?, ?>> accumulators) { super(env.getTaskInfo(), env.getUserClassLoader(), operator.getExecutionConfig(), accumulators, env.getDistributedCacheEntries(), operator.getMetricGroup()); this.operator = operator; this.taskEnvironment = env; this.streamConfig = new StreamConfig(env.getTaskConfiguration()); this.operatorUniqueID = operator.getOperatorID().toString(); }
private static <T> void setupSourceOperator( StreamSource<T, ?> operator, ExecutionConfig executionConfig, Environment env, ProcessingTimeService timeProvider) { StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateBackend(new MemoryStateBackend()); cfg.setTimeCharacteristic(TimeCharacteristic.EventTime); cfg.setOperatorID(new OperatorID()); StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class); when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE); StreamTask<?, ?> mockTask = mock(StreamTask.class); when(mockTask.getName()).thenReturn("Mock Task"); when(mockTask.getCheckpointLock()).thenReturn(new Object()); when(mockTask.getConfiguration()).thenReturn(cfg); when(mockTask.getEnvironment()).thenReturn(env); when(mockTask.getExecutionConfig()).thenReturn(executionConfig); when(mockTask.getAccumulatorMap()).thenReturn(Collections.<String, Accumulator<?, ?>>emptyMap()); when(mockTask.getStreamStatusMaintainer()).thenReturn(streamStatusMaintainer); doAnswer(new Answer<ProcessingTimeService>() { @Override public ProcessingTimeService answer(InvocationOnMock invocation) throws Throwable { if (timeProvider == null) { throw new RuntimeException("The time provider is null."); } return timeProvider; } }).when(mockTask).getProcessingTimeService(); operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class)); }
executionConfig.setAutoWatermarkInterval(watermarkInterval); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateBackend(new MemoryStateBackend());
@Test public void testStateBackendLoadingAndClosing() throws Exception { Configuration taskManagerConfig = new Configuration(); taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName()); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateKeySerializer(mock(TypeSerializer.class)); cfg.setOperatorID(new OperatorID(4711L, 42L)); TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction()); cfg.setStreamOperator(streamSource); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig); StateBackendTestSource.fail = false; task.startTaskThread(); // wait for clean termination task.getExecutingThread().join(); // ensure that the state backends and stream iterables are closed ... verify(TestStreamSource.operatorStateBackend).close(); verify(TestStreamSource.keyedStateBackend).close(); verify(TestStreamSource.rawOperatorStateInputs).close(); verify(TestStreamSource.rawKeyedStateInputs).close(); // ... and disposed verify(TestStreamSource.operatorStateBackend).dispose(); verify(TestStreamSource.keyedStateBackend).dispose(); assertEquals(ExecutionState.FINISHED, task.getExecutionState()); }
@Test public void testStateBackendClosingOnFailure() throws Exception { Configuration taskManagerConfig = new Configuration(); taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName()); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateKeySerializer(mock(TypeSerializer.class)); cfg.setOperatorID(new OperatorID(4711L, 42L)); TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction()); cfg.setStreamOperator(streamSource); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig); StateBackendTestSource.fail = true; task.startTaskThread(); // wait for clean termination task.getExecutingThread().join(); // ensure that the state backends and stream iterables are closed ... verify(TestStreamSource.operatorStateBackend).close(); verify(TestStreamSource.keyedStateBackend).close(); verify(TestStreamSource.rawOperatorStateInputs).close(); verify(TestStreamSource.rawKeyedStateInputs).close(); // ... and disposed verify(TestStreamSource.operatorStateBackend).dispose(); verify(TestStreamSource.keyedStateBackend).dispose(); assertEquals(ExecutionState.FAILED, task.getExecutionState()); }
public StreamTaskTestHarness( Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory, TypeInformation<OUT> outputType) { this.taskFactory = checkNotNull(taskFactory); this.memorySize = DEFAULT_MEMORY_MANAGER_SIZE; this.bufferSize = DEFAULT_NETWORK_BUFFER_SIZE; this.jobConfig = new Configuration(); this.taskConfig = new Configuration(); this.executionConfig = new ExecutionConfig(); streamConfig = new StreamConfig(taskConfig); outputSerializer = outputType.createSerializer(executionConfig); outputStreamRecordSerializer = new StreamElementSerializer<OUT>(outputSerializer); this.taskStateManager = new TestTaskStateManager(); }
assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapPrintVertex.getInputs().get(0).getSource().getResultType()); StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration()); StreamConfig mapConfig = new StreamConfig(mapPrintVertex.getConfiguration()); Map<Integer, StreamConfig> chainedConfigs = mapConfig.getTransitiveChainedTaskConfigs(getClass().getClassLoader()); StreamConfig printConfig = chainedConfigs.values().iterator().next();
@SafeVarargs private static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain( OneInputStreamOperator<T, T>... operators) { checkNotNull(operators); checkArgument(operators.length > 0); try (MockEnvironment env = MockEnvironment.builder().build()) { final StreamTask<?, ?> containingTask = new OneInputStreamTask<T, OneInputStreamOperator<T, T>>(env); final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class); final StreamConfig cfg = new StreamConfig(new Configuration()); final StreamOperator<?>[] ops = new StreamOperator<?>[operators.length]; // initial output goes to nowhere @SuppressWarnings({"unchecked", "rawtypes"}) WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>( new Output[0], statusProvider); // build the reverse operators array for (int i = 0; i < ops.length; i++) { OneInputStreamOperator<T, T> op = operators[ops.length - i - 1]; op.setup(containingTask, cfg, lastWriter); lastWriter = new ChainingOutput<>(op, statusProvider, null); ops[i] = op; } @SuppressWarnings("unchecked") final OP head = (OP) operators[0]; return new OperatorChain<>( ops, new RecordWriterOutput<?>[0], lastWriter, head); } }
@Test public void testCheckpointModeTranslation() { try { // with deactivated fault tolerance, the checkpoint mode should be at-least-once StreamExecutionEnvironment deactivated = getSimpleJob(); for (JobVertex vertex : deactivated.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } // with activated fault tolerance, the checkpoint mode should be by default exactly once StreamExecutionEnvironment activated = getSimpleJob(); activated.enableCheckpointing(1000L); for (JobVertex vertex : activated.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.EXACTLY_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } // explicitly setting the mode StreamExecutionEnvironment explicit = getSimpleJob(); explicit.enableCheckpointing(1000L, CheckpointingMode.AT_LEAST_ONCE); for (JobVertex vertex : explicit.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test public void testCancellationFailsWithBlockingLock() throws Exception { syncLatch = new OneShotLatch(); StreamConfig cfg = new StreamConfig(new Configuration()); Task task = createTask(CancelFailingTask.class, cfg, new Configuration()); // start the task and wait until it runs // execution state RUNNING is not enough, we need to wait until the stream task's run() method // is entered task.startTaskThread(); syncLatch.await(); // cancel the execution - this should lead to smooth shutdown task.cancelExecution(); task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
@Test public void testCancellationNotBlockedOnLock() throws Exception { syncLatch = new OneShotLatch(); StreamConfig cfg = new StreamConfig(new Configuration()); Task task = createTask(CancelLockingTask.class, cfg, new Configuration()); // start the task and wait until it runs // execution state RUNNING is not enough, we need to wait until the stream task's run() method // is entered task.startTaskThread(); syncLatch.await(); // cancel the execution - this should lead to smooth shutdown task.cancelExecution(); task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
/** * This test checks that cancel calls that are issued before the operator is * instantiated still lead to proper canceling. */ @Test public void testEarlyCanceling() throws Exception { final StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setOperatorID(new OperatorID(4711L, 42L)); cfg.setStreamOperator(new SlowlyDeserializingOperator()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions()); final Task task = createTask(SourceStreamTask.class, cfg, new Configuration(), taskManagerActions); final TaskExecutionState state = new TaskExecutionState( task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING); task.startTaskThread(); verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state)); // send a cancel. because the operator takes a long time to deserialize, this should // hit the task before the operator is deserialized task.cancelExecution(); task.getExecutingThread().join(); assertFalse("Task did not cancel", task.getExecutingThread().isAlive()); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); }
public <IN, OUT> StreamConfigChainer chain( OperatorID operatorID, OneInputStreamOperator<IN, OUT> operator, TypeSerializer<IN> inputSerializer, TypeSerializer<OUT> outputSerializer) { chainIndex++; tailConfig.setChainedOutputs(Collections.singletonList( new StreamEdge( new StreamNode(null, tailConfig.getChainIndex(), null, null, null, null, null, null), new StreamNode(null, chainIndex, null, null, null, null, null, null), 0, Collections.<String>emptyList(), null, null))); tailConfig = new StreamConfig(new Configuration()); tailConfig.setStreamOperator(checkNotNull(operator)); tailConfig.setOperatorID(checkNotNull(operatorID)); tailConfig.setTypeSerializerIn1(inputSerializer); tailConfig.setTypeSerializerOut(outputSerializer); tailConfig.setChainIndex(chainIndex); chainedConfigs.put(chainIndex, tailConfig); return this; }
@Test public void testLifeCycleFull() throws Exception { ACTUAL_ORDER_TRACKING.clear(); Configuration taskManagerConfig = new Configuration(); StreamConfig cfg = new StreamConfig(new Configuration()); MockSourceFunction srcFun = new MockSourceFunction(); cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true)); cfg.setOperatorID(new OperatorID()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig); task.startTaskThread(); LifecycleTrackingStreamSource.runStarted.await(); // wait for clean termination task.getExecutingThread().join(); assertEquals(ExecutionState.FINISHED, task.getExecutionState()); assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING); }
public void testOperatorClosingBeforeStopRunning() throws Throwable { Configuration taskConfiguration = new Configuration(); StreamConfig streamConfig = new StreamConfig(taskConfiguration); streamConfig.setStreamOperator(new BlockingCloseStreamOperator()); streamConfig.setOperatorID(new OperatorID());
@Test public void testLifeCycleCancel() throws Exception { ACTUAL_ORDER_TRACKING.clear(); Configuration taskManagerConfig = new Configuration(); StreamConfig cfg = new StreamConfig(new Configuration()); MockSourceFunction srcFun = new MockSourceFunction(); cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false)); cfg.setOperatorID(new OperatorID()); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig); task.startTaskThread(); LifecycleTrackingStreamSource.runStarted.await(); // this should cancel the task even though it is blocked on runFinished task.cancelExecution(); // wait for clean termination task.getExecutingThread().join(); assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING); }
final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration()); final AsyncWaitOperator<Integer, Integer> headOperator = operatorChainStreamConfig.getStreamOperator(AsyncWaitOperatorTest.class.getClassLoader());
private void testRestoreWithInterrupt(int mode) throws Exception { IN_RESTORE_LATCH.reset(); Configuration taskConfig = new Configuration(); StreamConfig cfg = new StreamConfig(taskConfig); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); switch (mode) { case OPERATOR_MANAGED: case OPERATOR_RAW: case KEYED_MANAGED: case KEYED_RAW: cfg.setStateKeySerializer(IntSerializer.INSTANCE); cfg.setStreamOperator(new StreamSource<>(new TestSource(mode))); break; default: throw new IllegalArgumentException(); } StreamStateHandle lockingHandle = new InterruptLockingStateHandle(); Task task = createTask(cfg, taskConfig, lockingHandle, mode); // start the task and wait until it is in "restore" task.startTaskThread(); IN_RESTORE_LATCH.await(); // trigger cancellation and signal to continue task.cancelExecution(); task.getExecutingThread().join(30000); if (task.getExecutionState() == ExecutionState.CANCELING) { fail("Task is stuck and not canceling"); } assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertNull(task.getFailureCause()); }