@Before public void setUp() throws Exception { CloseableRegistry closableRegistry = new CloseableRegistry(); CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(1024); KeyGroupRange keyGroupRange = new KeyGroupRange(0, 2); this.snapshotContext = new StateSnapshotContextSynchronousImpl(42, 4711, streamFactory, keyGroupRange, closableRegistry); }
timestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(Integer.MAX_VALUE)); fail("Exception expected."); } catch (Exception e) {
/** * Checks that the state snapshot context is closed after a successful snapshot operation. */ @Test public void testSnapshotMethod() throws Exception { final long checkpointId = 42L; final long timestamp = 1L; final CloseableRegistry closeableRegistry = new CloseableRegistry(); StateSnapshotContextSynchronousImpl context = spy(new StateSnapshotContextSynchronousImpl(0L, 0L)); whenNew(StateSnapshotContextSynchronousImpl.class).withAnyArguments().thenReturn(context); StreamTask<Void, AbstractStreamOperator<Void>> containingTask = mock(StreamTask.class); when(containingTask.getCancelables()).thenReturn(closeableRegistry); AbstractStreamOperator<Void> operator = mock(AbstractStreamOperator.class); when(operator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenCallRealMethod(); doReturn(containingTask).when(operator).getContainingTask(); operator.snapshotState( checkpointId, timestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(Integer.MAX_VALUE)); verify(context).close(); }
timestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(Integer.MAX_VALUE)); fail("Exception expected."); } catch (Exception e) {
CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(1024);
@Override public CheckpointStreamFactory resolveCheckpointStorageLocation( long checkpointId, CheckpointStorageLocationReference reference) throws IOException { // no matter where the checkpoint goes, we always return the storage location that stores // state inline with the state handles. return new MemCheckpointStreamFactory(maxStateSize); }
@Override public CheckpointStreamFactory createStreamFactory( JobID jobId, String operatorIdentifier) throws IOException { return new MemCheckpointStreamFactory(maxStateSize); }
@Override public CheckpointStreamFactory resolveCheckpointStorageLocation( long checkpointId, CheckpointStorageLocationReference reference) throws IOException { // no matter where the checkpoint goes, we always return the storage location that stores // state inline with the state handles. return new MemCheckpointStreamFactory(maxStateSize); }
@Override public CheckpointStreamFactory createSavepointStreamFactory( JobID jobId, String operatorIdentifier, String targetLocation) throws IOException { return new MemCheckpointStreamFactory(maxStateSize); }
@Override public CheckpointStreamFactory resolveCheckpointStorageLocation( long checkpointId, CheckpointStorageLocationReference reference) throws IOException { // no matter where the checkpoint goes, we always return the storage location that stores // state inline with the state handles. return new MemCheckpointStreamFactory(maxStateSize); }
/** * This is public so that we can use it when restoring a legacy snapshot * in {@code AbstractStreamOperatorTestHarness}. */ public static KeyGroupsStateHandle convertKeyedBackendState( HashMap<String, KvStateSnapshot<?, ?, ?, ?>> oldKeyedState, int parallelInstanceIdx, long checkpointID) throws Exception { if (null != oldKeyedState) { CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(MAX_SIZE); CheckpointStreamFactory.CheckpointStateOutputStream keyedStateOut = checkpointStreamFactory.createCheckpointStateOutputStream(checkpointID, 0L); try { final long offset = keyedStateOut.getPos(); InstantiationUtil.serializeObject(keyedStateOut, oldKeyedState); StreamStateHandle streamStateHandle = keyedStateOut.closeAndGetHandle(); keyedStateOut = null; // makes IOUtils.closeQuietly(...) ignore this if (null != streamStateHandle) { KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(parallelInstanceIdx, parallelInstanceIdx, new long[]{offset}); return new MigrationKeyGroupStateHandle(keyGroupRangeOffsets, streamStateHandle); } } finally { IOUtils.closeQuietly(keyedStateOut); } } return null; }