new StateObjectCollection<>(mergedManagedOperatorState), new StateObjectCollection<>(mergedRawOperatorState), new StateObjectCollection<>(mergedManagedKeyedState), new StateObjectCollection<>(mergedRawKeyedState));
new StateObjectCollection<>(managedOperatorState), new StateObjectCollection<>(rawOperatorState), new StateObjectCollection<>(managedKeyedState), new StateObjectCollection<>(rawKeyedState));
/** * Tests if there is an exception if all restore attempts are exhausted and failed. */ @Test public void testExceptionThrownIfAllRestoresFailed() throws Exception { CloseableRegistry closeableRegistry = new CloseableRegistry(); OperatorStateHandle firstFailHandle = mock(OperatorStateHandle.class); OperatorStateHandle secondFailHandle = mock(OperatorStateHandle.class); OperatorStateHandle thirdFailHandle = mock(OperatorStateHandle.class); List<StateObjectCollection<OperatorStateHandle>> sortedRestoreOptions = Arrays.asList( new StateObjectCollection<>(Collections.singletonList(firstFailHandle)), new StateObjectCollection<>(Collections.singletonList(secondFailHandle)), new StateObjectCollection<>(Collections.singletonList(thirdFailHandle))); BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> restorerProcedure = new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend"); try { restorerProcedure.createAndRestore(sortedRestoreOptions); Assert.fail(); } catch (Exception ignore) { } verify(firstFailHandle).openInputStream(); verify(secondFailHandle).openInputStream(); verify(thirdFailHandle).openInputStream(); }
Collections.singletonList(new StateObjectCollection<>(Collections.singletonList(blockingRestoreHandle)));
new StateObjectCollection<>(nullToEmptyCollection(localManagedOperatorState)), new StateObjectCollection<>(nullToEmptyCollection(localRawOperatorState)), new StateObjectCollection<>(nullToEmptyCollection(localManagedKeyGroupState)), new StateObjectCollection<>(nullToEmptyCollection(localRawKeyGroupState)));
new StateObjectCollection<>(operatorStateBackend), new StateObjectCollection<>(operatorStateStream), new StateObjectCollection<>(keyedStateFromBackend), new StateObjectCollection<>(keyedStateFromStream));
new StateObjectCollection<>(Collections.singletonList(firstFailHandle)), new StateObjectCollection<>(Collections.singletonList(secondSuccessHandle)), new StateObjectCollection<>(Collections.singletonList(thirdNotUsedHandle)));
new StateObjectCollection<>(operatorStateHandles), StateObjectCollection.empty(), new StateObjectCollection<>(keyedStateHandles));
public static <T extends StateObject> StateObjectCollection<T> singleton(T stateObject) { return new StateObjectCollection<>(Collections.singleton(stateObject)); }
public static <T extends StateObject> StateObjectCollection<T> singleton(T stateObject) { return new StateObjectCollection<>(Collections.singleton(stateObject)); }
public static <T extends StateObject> StateObjectCollection<T> singleton(T stateObject) { return new StateObjectCollection<>(Collections.singleton(stateObject)); }
public static OperatorSubtaskState operatorSubtaskStateFrom( OperatorInstanceID instanceID, Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState, Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState, Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) { if (!subManagedOperatorState.containsKey(instanceID) && !subRawOperatorState.containsKey(instanceID) && !subManagedKeyedState.containsKey(instanceID) && !subRawKeyedState.containsKey(instanceID)) { return new OperatorSubtaskState(); } if (!subManagedKeyedState.containsKey(instanceID)) { checkState(!subRawKeyedState.containsKey(instanceID)); } return new OperatorSubtaskState( new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList()))); }
public static OperatorSubtaskState operatorSubtaskStateFrom( OperatorInstanceID instanceID, Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState, Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState, Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) { if (!subManagedOperatorState.containsKey(instanceID) && !subRawOperatorState.containsKey(instanceID) && !subManagedKeyedState.containsKey(instanceID) && !subRawKeyedState.containsKey(instanceID)) { return new OperatorSubtaskState(); } if (!subManagedKeyedState.containsKey(instanceID)) { checkState(!subRawKeyedState.containsKey(instanceID)); } return new OperatorSubtaskState( new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList()))); }
public static OperatorSubtaskState operatorSubtaskStateFrom( OperatorInstanceID instanceID, Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState, Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState, Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) { if (!subManagedOperatorState.containsKey(instanceID) && !subRawOperatorState.containsKey(instanceID) && !subManagedKeyedState.containsKey(instanceID) && !subRawKeyedState.containsKey(instanceID)) { return new OperatorSubtaskState(); } if (!subManagedKeyedState.containsKey(instanceID)) { checkState(!subRawKeyedState.containsKey(instanceID)); } return new OperatorSubtaskState( new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList()))); }
private StateObjectCollection<OperatorStateHandle> transformSubtaskOpState(Path outDir, Integer subtaskId, StateObjectCollection<OperatorStateHandle> baseState) { if (transformer == null) { return baseState; } StateObjectCollection<OperatorStateHandle> opHandle = baseState; try (OperatorStateBackend opBackend = OperatorStateReader .restoreOperatorStateBackend(opHandle)) { transformer.accept(subtaskId, opBackend); OperatorStateHandle newSnapshot = opBackend .snapshot(checkpointId, System.currentTimeMillis(), new CheckpointStreamFactory() { @Override public CheckpointStateOutputStream createCheckpointStateOutputStream( CheckpointedStateScope scope) throws IOException { return new FileBasedStateOutputStream(outDir.getFileSystem(), new Path(outDir, String.valueOf(UUID.randomUUID()))); } }, null).get().getJobManagerOwnedSnapshot(); return new StateObjectCollection<>(Lists.newArrayList(newSnapshot)); } catch (Exception e) { throw new RuntimeException(e); } }
subtaskState.getRawOperatorState(), keepBaseKeyedStates ? subtaskState.getManagedKeyedState() : new StateObjectCollection<>( newKeyedHandle != null ? Lists.newArrayList(newKeyedHandle)