public OperatorSnapshotFinalizer( @Nonnull OperatorSnapshotFutures snapshotFutures) throws ExecutionException, InterruptedException { SnapshotResult<KeyedStateHandle> keyedManaged = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture()); SnapshotResult<KeyedStateHandle> keyedRaw = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture()); SnapshotResult<OperatorStateHandle> operatorManaged = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture()); SnapshotResult<OperatorStateHandle> operatorRaw = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture()); jobManagerOwnedState = new OperatorSubtaskState( operatorManaged.getJobManagerOwnedSnapshot(), operatorRaw.getJobManagerOwnedSnapshot(), keyedManaged.getJobManagerOwnedSnapshot(), keyedRaw.getJobManagerOwnedSnapshot() ); taskLocalState = new OperatorSubtaskState( operatorManaged.getTaskLocalSnapshot(), operatorRaw.getTaskLocalSnapshot(), keyedManaged.getTaskLocalSnapshot(), keyedRaw.getTaskLocalSnapshot() ); }
SnapshotResult.of(keyedManagedStateHandle); RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateManagedFuture = spy(DoneFuture.of(keyedStateManagedResult)); SnapshotResult.of(keyedRawStateHandle); RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateRawFuture = spy(DoneFuture.of(keyedStateRawResult)); SnapshotResult.of(operatorManagedStateHandle); RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateManagedFuture = spy(DoneFuture.of(operatorStateManagedResult)); SnapshotResult.of(operatorRawStateHandle); RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateRawFuture = spy(DoneFuture.of(operatorStateRawResult));
StateHandleDummyUtil.createNewOperatorStateHandle(2, random); SnapshotResult<KeyedStateHandle> snapKeyMan = SnapshotResult.withLocalState( StateHandleDummyUtil.deepDummyCopy(keyedTemplate), StateHandleDummyUtil.deepDummyCopy(keyedTemplate)); SnapshotResult<KeyedStateHandle> snapKeyRaw = SnapshotResult.withLocalState( StateHandleDummyUtil.deepDummyCopy(keyedTemplate), StateHandleDummyUtil.deepDummyCopy(keyedTemplate)); SnapshotResult<OperatorStateHandle> snapOpMan = SnapshotResult.withLocalState( StateHandleDummyUtil.deepDummyCopy(operatorTemplate), StateHandleDummyUtil.deepDummyCopy(operatorTemplate)); SnapshotResult<OperatorStateHandle> snapOpRaw = SnapshotResult.withLocalState( StateHandleDummyUtil.deepDummyCopy(operatorTemplate), StateHandleDummyUtil.deepDummyCopy(operatorTemplate)); Assert.assertTrue(checkResult(snapKeyMan.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getManagedKeyedState())); Assert.assertTrue(checkResult(snapKeyRaw.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getRawKeyedState())); Assert.assertTrue(checkResult(snapOpMan.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getManagedOperatorState())); Assert.assertTrue(checkResult(snapOpRaw.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getRawOperatorState())); Assert.assertTrue(checkResult(snapKeyMan.getTaskLocalSnapshot(), taskLocalState.getManagedKeyedState())); Assert.assertTrue(checkResult(snapKeyRaw.getTaskLocalSnapshot(), taskLocalState.getRawKeyedState())); Assert.assertTrue(checkResult(snapOpMan.getTaskLocalSnapshot(), taskLocalState.getManagedOperatorState())); Assert.assertTrue(checkResult(snapOpRaw.getTaskLocalSnapshot(), taskLocalState.getRawOperatorState()));
@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) { StreamStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot(); StreamStateHandle taskLocalSnapshot = snapshotResult.getTaskLocalSnapshot(); return SnapshotResult.withLocalState(jmKeyedState, localKeyedState); } else { return SnapshotResult.of(jmKeyedState); return SnapshotResult.empty();
@Nonnull @Override public SnapshotResult<StreamStateHandle> closeAndFinalizeCheckpointStreamResult() throws IOException { final StreamStateHandle primaryStreamStateHandle; try { primaryStreamStateHandle = outputStream.closeAndGetPrimaryHandle(); } catch (IOException primaryEx) { try { outputStream.close(); } catch (IOException closeEx) { primaryEx = ExceptionUtils.firstOrSuppressed(closeEx, primaryEx); } throw primaryEx; } StreamStateHandle secondaryStreamStateHandle = null; try { secondaryStreamStateHandle = outputStream.closeAndGetSecondaryHandle(); } catch (IOException secondaryEx) { LOG.warn("Exception from secondary/local checkpoint stream.", secondaryEx); } if (primaryStreamStateHandle != null) { if (secondaryStreamStateHandle != null) { return SnapshotResult.withLocalState(primaryStreamStateHandle, secondaryStreamStateHandle); } else { return SnapshotResult.of(primaryStreamStateHandle); } } else { return SnapshotResult.empty(); } }
public static <T extends StateObject> SnapshotResult<T> of(@Nullable T jobManagerState) { return jobManagerState != null ? new SnapshotResult<>(jobManagerState, null) : empty(); }
OperatorStateHandle secondSuccessHandle = spy(snapshotResult.getJobManagerOwnedSnapshot()); OperatorStateHandle thirdNotUsedHandle = mock(OperatorStateHandle.class);
private static <T extends StateObject> RunnableFuture<SnapshotResult<T>> createSnapshotResult(Class<T> clazz) { return DoneFuture.of(SnapshotResult.withLocalState(mock(clazz), mock(clazz))); } }
public static <T extends StateObject> SnapshotResult<T> withLocalState( @Nonnull T jobManagerState, @Nonnull T localState) { return new SnapshotResult<>(jobManagerState, localState); } }
@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) { StreamStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot(); StreamStateHandle taskLocalSnapshot = snapshotResult.getTaskLocalSnapshot(); return SnapshotResult.withLocalState(jmKeyedState, localKeyedState); } else { return SnapshotResult.of(jmKeyedState); return SnapshotResult.empty();
@Nonnull @Override public SnapshotResult<StreamStateHandle> closeAndFinalizeCheckpointStreamResult() throws IOException { final StreamStateHandle primaryStreamStateHandle; try { primaryStreamStateHandle = outputStream.closeAndGetPrimaryHandle(); } catch (IOException primaryEx) { try { outputStream.close(); } catch (IOException closeEx) { primaryEx = ExceptionUtils.firstOrSuppressed(closeEx, primaryEx); } throw primaryEx; } StreamStateHandle secondaryStreamStateHandle = null; try { secondaryStreamStateHandle = outputStream.closeAndGetSecondaryHandle(); } catch (IOException secondaryEx) { LOG.warn("Exception from secondary/local checkpoint stream.", secondaryEx); } if (primaryStreamStateHandle != null) { if (secondaryStreamStateHandle != null) { return SnapshotResult.withLocalState(primaryStreamStateHandle, secondaryStreamStateHandle); } else { return SnapshotResult.of(primaryStreamStateHandle); } } else { return SnapshotResult.empty(); } }
public static <T extends StateObject> SnapshotResult<T> of(@Nullable T jobManagerState) { return jobManagerState != null ? new SnapshotResult<>(jobManagerState, null) : empty(); }
private StateObjectCollection<OperatorStateHandle> transformSubtaskOpState(Path outDir, Integer subtaskId, StateObjectCollection<OperatorStateHandle> baseState) { if (transformer == null) { return baseState; } StateObjectCollection<OperatorStateHandle> opHandle = baseState; try (OperatorStateBackend opBackend = OperatorStateReader .restoreOperatorStateBackend(opHandle)) { transformer.accept(subtaskId, opBackend); OperatorStateHandle newSnapshot = opBackend .snapshot(checkpointId, System.currentTimeMillis(), new CheckpointStreamFactory() { @Override public CheckpointStateOutputStream createCheckpointStateOutputStream( CheckpointedStateScope scope) throws IOException { return new FileBasedStateOutputStream(outDir.getFileSystem(), new Path(outDir, String.valueOf(UUID.randomUUID()))); } }, null).get().getJobManagerOwnedSnapshot(); return new StateObjectCollection<>(Lists.newArrayList(newSnapshot)); } catch (Exception e) { throw new RuntimeException(e); } }
public static <T extends StateObject> SnapshotResult<T> withLocalState( @Nonnull T jobManagerState, @Nonnull T localState) { return new SnapshotResult<>(jobManagerState, localState); } }
@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) { StreamStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot(); StreamStateHandle taskLocalSnapshot = snapshotResult.getTaskLocalSnapshot(); return SnapshotResult.withLocalState(jmKeyedState, localKeyedState); } else { return SnapshotResult.of(jmKeyedState); return SnapshotResult.empty();
private static <T extends StateObject> void performCheck( Future<SnapshotResult<T>> resultFuture, StateObjectCollection<T> jmState, StateObjectCollection<T> tmState) { SnapshotResult<T> snapshotResult; try { snapshotResult = resultFuture.get(); } catch (Exception e) { throw new RuntimeException(e); } Assert.assertEquals( snapshotResult.getJobManagerOwnedSnapshot(), jmState.iterator().next()); Assert.assertEquals( snapshotResult.getTaskLocalSnapshot(), tmState.iterator().next()); }
@Nonnull @Override public SnapshotResult<StreamStateHandle> closeAndFinalizeCheckpointStreamResult() throws IOException { final StreamStateHandle primaryStreamStateHandle; try { primaryStreamStateHandle = outputStream.closeAndGetPrimaryHandle(); } catch (IOException primaryEx) { try { outputStream.close(); } catch (IOException closeEx) { primaryEx = ExceptionUtils.firstOrSuppressed(closeEx, primaryEx); } throw primaryEx; } StreamStateHandle secondaryStreamStateHandle = null; try { secondaryStreamStateHandle = outputStream.closeAndGetSecondaryHandle(); } catch (IOException secondaryEx) { LOG.warn("Exception from secondary/local checkpoint stream.", secondaryEx); } if (primaryStreamStateHandle != null) { if (secondaryStreamStateHandle != null) { return SnapshotResult.withLocalState(primaryStreamStateHandle, secondaryStreamStateHandle); } else { return SnapshotResult.of(primaryStreamStateHandle); } } else { return SnapshotResult.empty(); } }
@Nonnull @Override public final RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot( long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) throws Exception { if (kvStateInformation.isEmpty()) { if (LOG.isDebugEnabled()) { LOG.debug("Asynchronous RocksDB snapshot performed on empty keyed state at {}. Returning null.", timestamp); } return DoneFuture.of(SnapshotResult.empty()); } else { return doSnapshot(checkpointId, timestamp, streamFactory, checkpointOptions); } }
public static <T extends StateObject> SnapshotResult<T> of(@Nullable T jobManagerState) { return jobManagerState != null ? new SnapshotResult<>(jobManagerState, null) : empty(); }