Collection<KeyedStateHandle> managedKeyedState = state.getManagedKeyedState(); if (managedKeyedState != null) { dos.writeInt(managedKeyedState.size());
Collection<KeyedStateHandle> managedKeyedState = handle.getManagedKeyedState(); Collection<KeyedStateHandle> rawKeyedState = handle.getRawKeyedState();
assertEquals(StateObjectCollection.singleton(managedKeyedStateHandle), subtaskState.getManagedKeyedState()); assertEquals(StateObjectCollection.singleton(rawKeyedStateHandle), subtaskState.getRawKeyedState()); assertEquals(StateObjectCollection.singleton(managedOperatorStateHandle), subtaskState.getManagedOperatorState());
verify(keyedStateBackend).restore(eq(operatorSubtaskState.getManagedKeyedState()));
jmOperatorStateHandles.getManagedKeyedState(), localKeyGroupRange);
lastTaskManagerTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID); performCheck(osFuture.getKeyedStateManagedFuture(), jmState.getManagedKeyedState(), tmState.getManagedKeyedState()); performCheck(osFuture.getKeyedStateRawFuture(), jmState.getRawKeyedState(), tmState.getRawKeyedState()); performCheck(osFuture.getOperatorStateManagedFuture(), jmState.getManagedOperatorState(), tmState.getManagedOperatorState());
jobManagerOwnedState.getManagedKeyedState().discardState(); } else if (mode == JM_REMOVE_TM_RECOVERY) { taskLocalState.getManagedKeyedState().discardState();
Assert.assertTrue(checkResult(snapKeyMan.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getManagedKeyedState())); Assert.assertTrue(checkResult(snapKeyRaw.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getRawKeyedState())); Assert.assertTrue(checkResult(snapOpMan.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getManagedOperatorState())); Assert.assertTrue(checkResult(snapKeyMan.getTaskLocalSnapshot(), taskLocalState.getManagedKeyedState())); Assert.assertTrue(checkResult(snapKeyRaw.getTaskLocalSnapshot(), taskLocalState.getRawKeyedState())); Assert.assertTrue(checkResult(snapOpMan.getTaskLocalSnapshot(), taskLocalState.getManagedOperatorState()));
public static Optional<KeyedBackendSerializationProxy<?>> getKeyedBackendSerializationProxy(OperatorState opState) { try { KeyedStateHandle firstHandle = opState.getStates().iterator().next().getManagedKeyedState().iterator() .next(); if (firstHandle instanceof IncrementalKeyedStateHandle) { return Optional.of(getKeyedBackendSerializationProxy( ((IncrementalKeyedStateHandle) firstHandle).getMetaStateHandle())); } else { return Optional.of(getKeyedBackendSerializationProxy((StreamStateHandle) firstHandle)); } } catch (Exception e) { return Optional.empty(); } }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null && operatorState.getState(i).getManagedKeyedState() != null) { KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getManagedKeyedState().getIntersection(subtaskKeyGroupRange); if (intersectedKeyedStateHandle != null) { if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(); } subtaskKeyedStateHandles.add(intersectedKeyedStateHandle); } } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = new ArrayList<>(); for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> keyedStateHandles = operatorState.getState(i).getManagedKeyedState(); extractIntersectingState( keyedStateHandles, subtaskKeyGroupRange, subtaskKeyedStateHandles); } } return subtaskKeyedStateHandles; }
private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException { dos.writeLong(-1); int len = 0; dos.writeInt(len); OperatorStateHandle operatorStateBackend = extractSingleton(subtaskState.getManagedOperatorState()); len = operatorStateBackend != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateBackend, dos); } OperatorStateHandle operatorStateFromStream = extractSingleton(subtaskState.getRawOperatorState()); len = operatorStateFromStream != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateFromStream, dos); } KeyedStateHandle keyedStateBackend = extractSingleton(subtaskState.getManagedKeyedState()); serializeKeyedStateHandle(keyedStateBackend, dos); KeyedStateHandle keyedStateStream = extractSingleton(subtaskState.getRawKeyedState()); serializeKeyedStateHandle(keyedStateStream, dos); }
private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException { dos.writeLong(-1); int len = 0; dos.writeInt(len); OperatorStateHandle operatorStateBackend = extractSingleton(subtaskState.getManagedOperatorState()); len = operatorStateBackend != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateBackend, dos); } OperatorStateHandle operatorStateFromStream = extractSingleton(subtaskState.getRawOperatorState()); len = operatorStateFromStream != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateFromStream, dos); } KeyedStateHandle keyedStateBackend = extractSingleton(subtaskState.getManagedKeyedState()); serializeKeyedStateHandle(keyedStateBackend, dos); KeyedStateHandle keyedStateStream = extractSingleton(subtaskState.getRawKeyedState()); serializeKeyedStateHandle(keyedStateStream, dos); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } OperatorSubtaskState that = (OperatorSubtaskState) o; if (getStateSize() != that.getStateSize()) { return false; } if (!getManagedOperatorState().equals(that.getManagedOperatorState())) { return false; } if (!getRawOperatorState().equals(that.getRawOperatorState())) { return false; } if (!getManagedKeyedState().equals(that.getManagedKeyedState())) { return false; } return getRawKeyedState().equals(that.getRawKeyedState()); }
private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException { dos.writeLong(-1); int len = 0; dos.writeInt(len); OperatorStateHandle operatorStateBackend = extractSingleton(subtaskState.getManagedOperatorState()); len = operatorStateBackend != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateBackend, dos); } OperatorStateHandle operatorStateFromStream = extractSingleton(subtaskState.getRawOperatorState()); len = operatorStateFromStream != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateFromStream, dos); } KeyedStateHandle keyedStateBackend = extractSingleton(subtaskState.getManagedKeyedState()); serializeKeyedStateBackend(keyedStateBackend, dos); KeyedStateHandle keyedStateStream = extractSingleton(subtaskState.getRawKeyedState()); serializeRawKeyedStateHandle(keyedStateStream, dos); }
private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException { dos.writeLong(-1); int len = 0; dos.writeInt(len); OperatorStateHandle operatorStateBackend = extractSingleton(subtaskState.getManagedOperatorState()); len = operatorStateBackend != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateBackend, dos); } OperatorStateHandle operatorStateFromStream = extractSingleton(subtaskState.getRawOperatorState()); len = operatorStateFromStream != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateFromStream, dos); } KeyedStateHandle keyedStateBackend = extractSingleton(subtaskState.getManagedKeyedState()); serializeKeyedStateHandle(keyedStateBackend, dos); KeyedStateHandle keyedStateStream = extractSingleton(subtaskState.getRawKeyedState()); serializeKeyedStateHandle(keyedStateStream, dos); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } OperatorSubtaskState that = (OperatorSubtaskState) o; if (getStateSize() != that.getStateSize()) { return false; } if (!getManagedOperatorState().equals(that.getManagedOperatorState())) { return false; } if (!getRawOperatorState().equals(that.getRawOperatorState())) { return false; } if (!getManagedKeyedState().equals(that.getManagedKeyedState())) { return false; } return getRawKeyedState().equals(that.getRawKeyedState()); }
@Override public int hashCode() { int result = getManagedOperatorState().hashCode(); result = 31 * result + getRawOperatorState().hashCode(); result = 31 * result + getManagedKeyedState().hashCode(); result = 31 * result + getRawKeyedState().hashCode(); result = 31 * result + (int) (getStateSize() ^ (getStateSize() >>> 32)); return result; }
@Override public int hashCode() { int result = getManagedOperatorState().hashCode(); result = 31 * result + getRawOperatorState().hashCode(); result = 31 * result + getManagedKeyedState().hashCode(); result = 31 * result + getRawKeyedState().hashCode(); result = 31 * result + (int) (getStateSize() ^ (getStateSize() >>> 32)); return result; }
@Override public int hashCode() { int result = getManagedOperatorState().hashCode(); result = 31 * result + getRawOperatorState().hashCode(); result = 31 * result + getManagedKeyedState().hashCode(); result = 31 * result + getRawKeyedState().hashCode(); result = 31 * result + (int) (getStateSize() ^ (getStateSize() >>> 32)); return result; }