@Override public void snapshotState(StateSnapshotContext context) throws Exception { ListState<Integer> partitionableState = getOperatorStateBackend().getListState(TEST_DESCRIPTOR); partitionableState.clear(); partitionableState.add(42); partitionableState.add(4711); ++numberSnapshotCalls; }
ListState<Integer> listState = originalBackend.getListState(stateDescriptor); originalBackend.snapshot(0L, 0L, checkpointStreamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()); originalBackend.close(); originalBackend.dispose(); verifyZeroInteractions(thirdNotUsedHandle); ListState<Integer> listState = restoredBackend.getListState(stateDescriptor); restoredBackend.close(); restoredBackend.dispose();
private static boolean trySnapshotFunctionState( StateSnapshotContext context, OperatorStateBackend backend, Function userFunction) throws Exception { if (userFunction instanceof CheckpointedFunction) { ((CheckpointedFunction) userFunction).snapshotState(context); return true; } if (userFunction instanceof ListCheckpointed) { @SuppressWarnings("unchecked") List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction). snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp()); ListState<Serializable> listState = backend. getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME); listState.clear(); if (null != partitionableState) { try { for (Serializable statePartition : partitionableState) { listState.add(statePartition); } } catch (Exception e) { listState.clear(); throw new Exception("Could not write partitionable state to operator " + "state backend.", e); } } return true; } return false; }
@Override public OperatorStateBackend createOperatorStateBackend(Environment env, String operatorIdentifier) throws Exception { OperatorStateBackend operatorStateBackend = mock(OperatorStateBackend.class); when(operatorStateBackend.snapshot(anyLong(), anyLong(), any(CheckpointStreamFactory.class), any(CheckpointOptions.class))) .thenReturn(new FutureTask<>(new BlockingCallable())); return operatorStateBackend; } }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); this.broadcastStates = new HashMap<>(broadcastStateDescriptors.size()); for (MapStateDescriptor<?, ?> descriptor: broadcastStateDescriptors) { broadcastStates.put(descriptor, getOperatorStateBackend().getBroadcastState(descriptor)); } rwContext = new ReadWriteContextImpl(getExecutionConfig(), userFunction, broadcastStates, getProcessingTimeService()); rContext = new ReadOnlyContextImpl(getExecutionConfig(), userFunction, broadcastStates, getProcessingTimeService()); }
IOUtils.closeQuietly(operatorStateBackend); operatorStateBackend.dispose();
verify(operatorStateBackend).restore(eq(operatorSubtaskState.getManagedOperatorState())); verify(keyedStateBackend).restore(eq(operatorSubtaskState.getManagedKeyedState()));
@Test public void testStateBackendLoadingAndClosing() throws Exception { Configuration taskManagerConfig = new Configuration(); taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName()); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateKeySerializer(mock(TypeSerializer.class)); cfg.setOperatorID(new OperatorID(4711L, 42L)); TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction()); cfg.setStreamOperator(streamSource); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig); StateBackendTestSource.fail = false; task.startTaskThread(); // wait for clean termination task.getExecutingThread().join(); // ensure that the state backends and stream iterables are closed ... verify(TestStreamSource.operatorStateBackend).close(); verify(TestStreamSource.keyedStateBackend).close(); verify(TestStreamSource.rawOperatorStateInputs).close(); verify(TestStreamSource.rawKeyedStateInputs).close(); // ... and disposed verify(TestStreamSource.operatorStateBackend).dispose(); verify(TestStreamSource.keyedStateBackend).dispose(); assertEquals(ExecutionState.FINISHED, task.getExecutionState()); }
operatorStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
@Override public void open() throws Exception { super.open(); InternalTimerService<VoidNamespace> internalTimerService = getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this); TimerService timerService = new SimpleTimerService(internalTimerService); collector = new TimestampedCollector<>(output); this.broadcastStates = new HashMap<>(broadcastStateDescriptors.size()); for (MapStateDescriptor<?, ?> descriptor: broadcastStateDescriptors) { broadcastStates.put(descriptor, getOperatorStateBackend().getBroadcastState(descriptor)); } rwContext = new ReadWriteContextImpl(getExecutionConfig(), getKeyedStateBackend(), userFunction, broadcastStates, timerService); rContext = new ReadOnlyContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService); onTimerContext = new OnTimerContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService); }
/** * This method is called at the very end of the operator's life, both in the case of a successful * completion of the operation, and in the case of a failure and canceling. * * <p>This method is expected to make a thorough effort to release all resources * that the operator has acquired. */ @Override public void dispose() throws Exception { if (operatorStateBackend != null) { operatorStateBackend.dispose(); } if (keyedStateBackend != null) { keyedStateBackend.dispose(); } }
public OperatorStateBackend createOperatorStateBackend( StreamOperator<?> op, Collection<OperatorStateHandle> restoreStateHandles) throws Exception { Environment env = getEnvironment(); String opId = createOperatorIdentifier(op, getConfiguration().getVertexID()); OperatorStateBackend operatorStateBackend = stateBackend.createOperatorStateBackend(env, opId); // let operator state backend participate in the operator lifecycle, i.e. make it responsive to cancelation cancelables.registerClosable(operatorStateBackend); // restore if we have some old state if (null != restoreStateHandles) { operatorStateBackend.restore(restoreStateHandles); } return operatorStateBackend; }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); ListState<StreamElement> partitionableState = getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer)); partitionableState.clear(); Collection<StreamElementQueueEntry<?>> values = queue.values(); try { for (StreamElementQueueEntry<?> value : values) { partitionableState.add(value.getStreamElement()); } // add the pending stream element queue entry if the stream element queue is currently full if (pendingStreamElementQueueEntry != null) { partitionableState.add(pendingStreamElementQueueEntry.getStreamElement()); } } catch (Exception e) { partitionableState.clear(); throw new Exception("Could not add stream element queue entries to operator state " + "backend of operator " + getOperatorName() + '.', e); } }
@Test public void testStateBackendClosingOnFailure() throws Exception { Configuration taskManagerConfig = new Configuration(); taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName()); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateKeySerializer(mock(TypeSerializer.class)); cfg.setOperatorID(new OperatorID(4711L, 42L)); TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction()); cfg.setStreamOperator(streamSource); cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig); StateBackendTestSource.fail = true; task.startTaskThread(); // wait for clean termination task.getExecutingThread().join(); // ensure that the state backends and stream iterables are closed ... verify(TestStreamSource.operatorStateBackend).close(); verify(TestStreamSource.keyedStateBackend).close(); verify(TestStreamSource.rawOperatorStateInputs).close(); verify(TestStreamSource.rawKeyedStateInputs).close(); // ... and disposed verify(TestStreamSource.operatorStateBackend).dispose(); verify(TestStreamSource.keyedStateBackend).dispose(); assertEquals(ExecutionState.FAILED, task.getExecutionState()); }
private StateObjectCollection<OperatorStateHandle> transformSubtaskOpState(Path outDir, Integer subtaskId, StateObjectCollection<OperatorStateHandle> baseState) { if (transformer == null) { return baseState; } StateObjectCollection<OperatorStateHandle> opHandle = baseState; try (OperatorStateBackend opBackend = OperatorStateReader .restoreOperatorStateBackend(opHandle)) { transformer.accept(subtaskId, opBackend); OperatorStateHandle newSnapshot = opBackend .snapshot(checkpointId, System.currentTimeMillis(), new CheckpointStreamFactory() { @Override public CheckpointStateOutputStream createCheckpointStateOutputStream( CheckpointedStateScope scope) throws IOException { return new FileBasedStateOutputStream(outDir.getFileSystem(), new Path(outDir, String.valueOf(UUID.randomUUID()))); } }, null).get().getJobManagerOwnedSnapshot(); return new StateObjectCollection<>(Lists.newArrayList(newSnapshot)); } catch (Exception e) { throw new RuntimeException(e); } }
/** * Read the serializableListState stored in the checkpoint for the given * operator subtask */ public List<Serializable> getSerializableListState(int subtask) throws Exception { OperatorStateBackend backend = createOperatorStateBackendFromSnapshot(subtask); @SuppressWarnings("deprecation") ListState<Serializable> listState = backend .getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME); List<Serializable> list = new ArrayList<>(); for (Serializable serializable : listState.get()) { list.add(serializable); } return list; }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); this.broadcastStates = new HashMap<>(broadcastStateDescriptors.size()); for (MapStateDescriptor<?, ?> descriptor: broadcastStateDescriptors) { broadcastStates.put(descriptor, getOperatorStateBackend().getBroadcastState(descriptor)); } rwContext = new ReadWriteContextImpl(getExecutionConfig(), userFunction, broadcastStates, getProcessingTimeService()); rContext = new ReadOnlyContextImpl(getExecutionConfig(), userFunction, broadcastStates, getProcessingTimeService()); }
IOUtils.closeQuietly(operatorStateBackend); operatorStateBackend.dispose();