private void initializeState() throws Exception { StreamOperator<?>[] allOperators = operatorChain.getAllOperators(); for (StreamOperator<?> operator : allOperators) { if (null != operator) { operator.initializeState(); } } }
/** * Execute {@link StreamOperator#open()} of each operator in the chain of this * {@link StreamTask}. Opening happens from <b>tail to head</b> operator in the chain, contrary * to {@link StreamOperator#close()} which happens <b>head to tail</b> * (see {@link #closeAllOperators()}. */ private void openAllOperators() throws Exception { for (StreamOperator<?> operator : operatorChain.getAllOperators()) { if (operator != null) { operator.open(); } } }
@SuppressWarnings("deprecation") private void checkpointStreamOperator(StreamOperator<?> op) throws Exception { if (null != op) { OperatorSnapshotFutures snapshotInProgress = op.snapshotState( checkpointMetaData.getCheckpointId(), checkpointMetaData.getTimestamp(), checkpointOptions, storageLocation); operatorSnapshotsInProgress.put(op.getOperatorID(), snapshotInProgress); } }
headOperator.setup(containingTask, configuration, output); headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, output.getWatermarkGauge());
/** * Execute {@link StreamOperator#dispose()} of each operator in the chain of this * {@link StreamTask}. Disposing happens from <b>tail to head</b> operator in the chain. */ private void tryDisposeAllOperators() throws Exception { for (StreamOperator<?> operator : operatorChain.getAllOperators()) { if (operator != null) { operator.dispose(); } } }
private boolean isChainable(StreamEdge edge, boolean isChainingEnabled, StreamGraph streamGraph) { StreamNode upStreamVertex = streamGraph.getSourceVertex(edge); StreamNode downStreamVertex = streamGraph.getTargetVertex(edge); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && isChainingEnabled; } }
/** * Execute {@link StreamOperator#close()} of each operator in the chain of this * {@link StreamTask}. Closing happens from <b>head to tail</b> operator in the chain, * contrary to {@link StreamOperator#open()} which happens <b>tail to head</b> * (see {@link #openAllOperators()}. */ private void closeAllOperators() throws Exception { // We need to close them first to last, since upstream operators in the chain might emit // elements in their close methods. StreamOperator<?>[] allOperators = operatorChain.getAllOperators(); for (int i = allOperators.length - 1; i >= 0; i--) { StreamOperator<?> operator = allOperators[i]; if (operator != null) { operator.close(); } } }
jmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), processedJmOpSubtaskState); tmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), tmOperatorStateHandles); taskStateManager.setTaskManagerTaskStateSnapshotsByCheckpointId( Collections.singletonMap(0L, tmTaskStateSnapshot)); operator.initializeState(); initializeCalled = true;
/** * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#notifyCheckpointComplete(long)} ()}. */ public void notifyOfCompletedCheckpoint(long checkpointId) throws Exception { operator.notifyCheckpointComplete(checkpointId); }
/** * Calls {@link StreamOperator#setup(StreamTask, StreamConfig, Output)} ()}. */ public void setup(TypeSerializer<OUT> outputSerializer) { if (!setupCalled) { this.streamTaskStateInitializer = createStreamTaskStateManager(environment, stateBackend, processingTimeService); when(mockTask.createStreamTaskStateInitializer()).thenReturn(streamTaskStateInitializer); operator.setup(mockTask, config, new MockOutput(outputSerializer)); setupCalled = true; } }
@SuppressWarnings("deprecation") private void checkpointStreamOperator(StreamOperator<?> op) throws Exception { if (null != op) { // first call the legacy checkpoint code paths nonPartitionedStates.add(op.snapshotLegacyOperatorState( checkpointMetaData.getCheckpointId(), checkpointMetaData.getTimestamp(), checkpointOptions)); OperatorSnapshotResult snapshotInProgress = op.snapshotState( checkpointMetaData.getCheckpointId(), checkpointMetaData.getTimestamp(), checkpointOptions); snapshotInProgressList.add(snapshotInProgress); } else { nonPartitionedStates.add(null); OperatorSnapshotResult emptySnapshotInProgress = new OperatorSnapshotResult(); snapshotInProgressList.add(emptySnapshotInProgress); } }
/** * Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions, org.apache.flink.runtime.state.CheckpointStreamFactory)}. */ public OperatorSnapshotFinalizer snapshotWithLocalState(long checkpointId, long timestamp) throws Exception { OperatorSnapshotFutures operatorStateResult = operator.snapshotState( checkpointId, timestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), checkpointStorage.resolveCheckpointStorageLocation(checkpointId, CheckpointStorageLocationReference.getDefault())); return new OperatorSnapshotFinalizer(operatorStateResult); }
when(streamOperator1.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenReturn(operatorSnapshotResult1); when(streamOperator2.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenReturn(operatorSnapshotResult2); when(streamOperator3.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenReturn(operatorSnapshotResult3); when(streamOperator1.getOperatorID()).thenReturn(operatorID1); when(streamOperator2.getOperatorID()).thenReturn(operatorID2); when(streamOperator3.getOperatorID()).thenReturn(operatorID3);
/** * Execute @link StreamOperator#dispose()} of each operator in the chain of this * {@link StreamTask}. Disposing happens from <b>tail to head</b> operator in the chain. * * <p>The difference with the {@link #tryDisposeAllOperators()} is that in case of an * exception, this method catches it and logs the message. */ private void disposeAllOperators() { if (operatorChain != null) { for (StreamOperator<?> operator : operatorChain.getAllOperators()) { try { if (operator != null) { operator.dispose(); } } catch (Throwable t) { LOG.error("Error during disposal of stream operator.", t); } } } }
headOperator.setup(containingTask, configuration, output); headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, output.getWatermarkGauge());
public static boolean isChainable(StreamEdge edge, StreamGraph streamGraph) { StreamNode upStreamVertex = streamGraph.getSourceVertex(edge); StreamNode downStreamVertex = streamGraph.getTargetVertex(edge); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && streamGraph.isChainingEnabled(); }
/** * Execute {@link StreamOperator#close()} of each operator in the chain of this * {@link StreamTask}. Closing happens from <b>head to tail</b> operator in the chain, * contrary to {@link StreamOperator#open()} which happens <b>tail to head</b> * (see {@link #openAllOperators()}. */ private void closeAllOperators() throws Exception { // We need to close them first to last, since upstream operators in the chain might emit // elements in their close methods. StreamOperator<?>[] allOperators = operatorChain.getAllOperators(); for (int i = allOperators.length - 1; i >= 0; i--) { StreamOperator<?> operator = allOperators[i]; if (operator != null) { operator.close(); } } }
@Override public void notifyCheckpointComplete(long checkpointId) throws Exception { synchronized (lock) { if (isRunning) { LOG.debug("Notification of complete checkpoint for task {}", getName()); for (StreamOperator<?> operator : operatorChain.getAllOperators()) { if (operator != null) { operator.notifyCheckpointComplete(checkpointId); } } } else { LOG.debug("Ignoring notification of complete checkpoint for not-running task {}", getName()); } } }
headOperator.setup(containingTask, configuration, output);