Collection<KeyedStateHandle> rawKeyedState = state.getRawKeyedState(); if (rawKeyedState != null) { dos.writeInt(rawKeyedState.size());
Collection<OperatorStateHandle> rawOperatorState = handle.getRawOperatorState(); Collection<KeyedStateHandle> managedKeyedState = handle.getManagedKeyedState(); Collection<KeyedStateHandle> rawKeyedState = handle.getRawKeyedState();
assertEquals(StateObjectCollection.singleton(rawKeyedStateHandle), subtaskState.getRawKeyedState()); assertEquals(StateObjectCollection.singleton(managedOperatorStateHandle), subtaskState.getManagedOperatorState()); assertEquals(StateObjectCollection.singleton(rawOperatorStateHandle), subtaskState.getRawOperatorState());
jmOperatorStateHandles.getRawKeyedState(), localKeyGroupRange);
performCheck(osFuture.getKeyedStateRawFuture(), jmState.getRawKeyedState(), tmState.getRawKeyedState()); performCheck(osFuture.getOperatorStateManagedFuture(), jmState.getManagedOperatorState(), tmState.getManagedOperatorState()); performCheck(osFuture.getOperatorStateRawFuture(), jmState.getRawOperatorState(), tmState.getRawOperatorState());
Assert.assertTrue(checkResult(snapKeyRaw.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getRawKeyedState())); Assert.assertTrue(checkResult(snapOpMan.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getManagedOperatorState())); Assert.assertTrue(checkResult(snapOpRaw.getJobManagerOwnedSnapshot(), jobManagerOwnedState.getRawOperatorState())); Assert.assertTrue(checkResult(snapKeyRaw.getTaskLocalSnapshot(), taskLocalState.getRawKeyedState())); Assert.assertTrue(checkResult(snapOpMan.getTaskLocalSnapshot(), taskLocalState.getManagedOperatorState())); Assert.assertTrue(checkResult(snapOpRaw.getTaskLocalSnapshot(), taskLocalState.getRawOperatorState()));
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null && operatorState.getState(i).getRawKeyedState() != null) { KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getRawKeyedState().getIntersection(subtaskKeyGroupRange); if (intersectedKeyedStateHandle != null) { if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(); } subtaskKeyedStateHandles.add(intersectedKeyedStateHandle); } } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> extractedKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> rawKeyedState = operatorState.getState(i).getRawKeyedState(); if (extractedKeyedStateHandles == null) { extractedKeyedStateHandles = new ArrayList<>(parallelism * rawKeyedState.size()); } extractIntersectingState( rawKeyedState, subtaskKeyGroupRange, extractedKeyedStateHandles); } } return extractedKeyedStateHandles; }
private Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> reAssignSubKeyedStates( OperatorState operatorState, List<KeyGroupRange> keyGroupPartitions, int subTaskIndex, int newParallelism, int oldParallelism) { List<KeyedStateHandle> subManagedKeyedState; List<KeyedStateHandle> subRawKeyedState; if (newParallelism == oldParallelism) { if (operatorState.getState(subTaskIndex) != null) { subManagedKeyedState = operatorState.getState(subTaskIndex).getManagedKeyedState().asList(); subRawKeyedState = operatorState.getState(subTaskIndex).getRawKeyedState().asList(); } else { subManagedKeyedState = Collections.emptyList(); subRawKeyedState = Collections.emptyList(); } } else { subManagedKeyedState = getManagedKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); subRawKeyedState = getRawKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); } if (subManagedKeyedState.isEmpty() && subRawKeyedState.isEmpty()) { return new Tuple2<>(Collections.emptyList(), Collections.emptyList()); } else { return new Tuple2<>(subManagedKeyedState, subRawKeyedState); } }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> extractedKeyedStateHandles = new ArrayList<>(); for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> rawKeyedState = operatorState.getState(i).getRawKeyedState(); extractIntersectingState( rawKeyedState, subtaskKeyGroupRange, extractedKeyedStateHandles); } } return extractedKeyedStateHandles; }
private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException { dos.writeLong(-1); int len = 0; dos.writeInt(len); OperatorStateHandle operatorStateBackend = extractSingleton(subtaskState.getManagedOperatorState()); len = operatorStateBackend != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateBackend, dos); } OperatorStateHandle operatorStateFromStream = extractSingleton(subtaskState.getRawOperatorState()); len = operatorStateFromStream != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateFromStream, dos); } KeyedStateHandle keyedStateBackend = extractSingleton(subtaskState.getManagedKeyedState()); serializeKeyedStateHandle(keyedStateBackend, dos); KeyedStateHandle keyedStateStream = extractSingleton(subtaskState.getRawKeyedState()); serializeKeyedStateHandle(keyedStateStream, dos); }
private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException { dos.writeLong(-1); int len = 0; dos.writeInt(len); OperatorStateHandle operatorStateBackend = extractSingleton(subtaskState.getManagedOperatorState()); len = operatorStateBackend != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateBackend, dos); } OperatorStateHandle operatorStateFromStream = extractSingleton(subtaskState.getRawOperatorState()); len = operatorStateFromStream != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateFromStream, dos); } KeyedStateHandle keyedStateBackend = extractSingleton(subtaskState.getManagedKeyedState()); serializeKeyedStateHandle(keyedStateBackend, dos); KeyedStateHandle keyedStateStream = extractSingleton(subtaskState.getRawKeyedState()); serializeKeyedStateHandle(keyedStateStream, dos); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } OperatorSubtaskState that = (OperatorSubtaskState) o; if (getStateSize() != that.getStateSize()) { return false; } if (!getManagedOperatorState().equals(that.getManagedOperatorState())) { return false; } if (!getRawOperatorState().equals(that.getRawOperatorState())) { return false; } if (!getManagedKeyedState().equals(that.getManagedKeyedState())) { return false; } return getRawKeyedState().equals(that.getRawKeyedState()); }
private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException { dos.writeLong(-1); int len = 0; dos.writeInt(len); OperatorStateHandle operatorStateBackend = extractSingleton(subtaskState.getManagedOperatorState()); len = operatorStateBackend != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateBackend, dos); } OperatorStateHandle operatorStateFromStream = extractSingleton(subtaskState.getRawOperatorState()); len = operatorStateFromStream != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateFromStream, dos); } KeyedStateHandle keyedStateBackend = extractSingleton(subtaskState.getManagedKeyedState()); serializeKeyedStateBackend(keyedStateBackend, dos); KeyedStateHandle keyedStateStream = extractSingleton(subtaskState.getRawKeyedState()); serializeRawKeyedStateHandle(keyedStateStream, dos); }
private static void serializeSubtaskState(OperatorSubtaskState subtaskState, DataOutputStream dos) throws IOException { dos.writeLong(-1); int len = 0; dos.writeInt(len); OperatorStateHandle operatorStateBackend = extractSingleton(subtaskState.getManagedOperatorState()); len = operatorStateBackend != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateBackend, dos); } OperatorStateHandle operatorStateFromStream = extractSingleton(subtaskState.getRawOperatorState()); len = operatorStateFromStream != null ? 1 : 0; dos.writeInt(len); if (len == 1) { serializeOperatorStateHandle(operatorStateFromStream, dos); } KeyedStateHandle keyedStateBackend = extractSingleton(subtaskState.getManagedKeyedState()); serializeKeyedStateHandle(keyedStateBackend, dos); KeyedStateHandle keyedStateStream = extractSingleton(subtaskState.getRawKeyedState()); serializeKeyedStateHandle(keyedStateStream, dos); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } OperatorSubtaskState that = (OperatorSubtaskState) o; if (getStateSize() != that.getStateSize()) { return false; } if (!getManagedOperatorState().equals(that.getManagedOperatorState())) { return false; } if (!getRawOperatorState().equals(that.getRawOperatorState())) { return false; } if (!getManagedKeyedState().equals(that.getManagedKeyedState())) { return false; } return getRawKeyedState().equals(that.getRawKeyedState()); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } OperatorSubtaskState that = (OperatorSubtaskState) o; if (getStateSize() != that.getStateSize()) { return false; } if (!getManagedOperatorState().equals(that.getManagedOperatorState())) { return false; } if (!getRawOperatorState().equals(that.getRawOperatorState())) { return false; } if (!getManagedKeyedState().equals(that.getManagedKeyedState())) { return false; } return getRawKeyedState().equals(that.getRawKeyedState()); }
@Override public int hashCode() { int result = getManagedOperatorState().hashCode(); result = 31 * result + getRawOperatorState().hashCode(); result = 31 * result + getManagedKeyedState().hashCode(); result = 31 * result + getRawKeyedState().hashCode(); result = 31 * result + (int) (getStateSize() ^ (getStateSize() >>> 32)); return result; }
@Override public int hashCode() { int result = getManagedOperatorState().hashCode(); result = 31 * result + getRawOperatorState().hashCode(); result = 31 * result + getManagedKeyedState().hashCode(); result = 31 * result + getRawKeyedState().hashCode(); result = 31 * result + (int) (getStateSize() ^ (getStateSize() >>> 32)); return result; }
@Override public int hashCode() { int result = getManagedOperatorState().hashCode(); result = 31 * result + getRawOperatorState().hashCode(); result = 31 * result + getManagedKeyedState().hashCode(); result = 31 * result + getRawKeyedState().hashCode(); result = 31 * result + (int) (getStateSize() ^ (getStateSize() >>> 32)); return result; }