private void reAssignSubNonPartitionedStates( OperatorState operatorState, int subTaskIndex, int newParallelism, int oldParallelism, List<StreamStateHandle> subNonPartitionableState) { if (oldParallelism == newParallelism) { if (operatorState.getState(subTaskIndex) != null) { subNonPartitionableState.add(operatorState.getState(subTaskIndex).getLegacyOperatorState()); } else { subNonPartitionableState.add(null); } } else { subNonPartitionableState.add(null); } }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null && operatorState.getState(i).getManagedKeyedState() != null) { KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getManagedKeyedState().getIntersection(subtaskKeyGroupRange); if (intersectedKeyedStateHandle != null) { if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(); } subtaskKeyedStateHandles.add(intersectedKeyedStateHandle); } } } return subtaskKeyedStateHandles; }
/** * Restores the OperatorStateBackend corresponding to the given subtask. The * backend is completely restored in-memory. */ public OperatorStateBackend createOperatorStateBackendFromSnapshot(int subtask) throws Exception { return restoreOperatorStateBackend(opState.getState(subtask).getManagedOperatorState()); }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null && operatorState.getState(i).getRawKeyedState() != null) { KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getRawKeyedState().getIntersection(subtaskKeyGroupRange); if (intersectedKeyedStateHandle != null) { if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(); } subtaskKeyedStateHandles.add(intersectedKeyedStateHandle); } } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = new ArrayList<>(); for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> keyedStateHandles = operatorState.getState(i).getManagedKeyedState(); extractIntersectingState( keyedStateHandles, subtaskKeyGroupRange, subtaskKeyedStateHandles); } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> extractedKeyedStateHandles = new ArrayList<>(); for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> rawKeyedState = operatorState.getState(i).getRawKeyedState(); extractIntersectingState( rawKeyedState, subtaskKeyGroupRange, extractedKeyedStateHandles); } } return extractedKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> extractedKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> rawKeyedState = operatorState.getState(i).getRawKeyedState(); if (extractedKeyedStateHandles == null) { extractedKeyedStateHandles = new ArrayList<>(parallelism * rawKeyedState.size()); } extractIntersectingState( rawKeyedState, subtaskKeyGroupRange, extractedKeyedStateHandles); } } return extractedKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> keyedStateHandles = operatorState.getState(i).getManagedKeyedState(); if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(parallelism * keyedStateHandles.size()); } extractIntersectingState( keyedStateHandles, subtaskKeyGroupRange, subtaskKeyedStateHandles); } } return subtaskKeyedStateHandles; }
private Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> reAssignSubKeyedStates( OperatorState operatorState, List<KeyGroupRange> keyGroupPartitions, int subTaskIndex, int newParallelism, int oldParallelism) { Collection<KeyedStateHandle> subManagedKeyedState; Collection<KeyedStateHandle> subRawKeyedState; if (newParallelism == oldParallelism) { if (operatorState.getState(subTaskIndex) != null) { subManagedKeyedState = operatorState.getState(subTaskIndex).getManagedKeyedState(); subRawKeyedState = operatorState.getState(subTaskIndex).getRawKeyedState(); } else { subManagedKeyedState = Collections.emptyList(); subRawKeyedState = Collections.emptyList(); } } else { subManagedKeyedState = getManagedKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); subRawKeyedState = getRawKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); } if (subManagedKeyedState.isEmpty() && subRawKeyedState.isEmpty()) { return new Tuple2<>(Collections.emptyList(), Collections.emptyList()); } else { return new Tuple2<>(subManagedKeyedState, subRawKeyedState); } }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> keyedStateHandles = operatorState.getState(i).getManagedKeyedState(); if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(parallelism * keyedStateHandles.size()); } extractIntersectingState( keyedStateHandles, subtaskKeyGroupRange, subtaskKeyedStateHandles); } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> extractedKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> rawKeyedState = operatorState.getState(i).getRawKeyedState(); if (extractedKeyedStateHandles == null) { extractedKeyedStateHandles = new ArrayList<>(parallelism * rawKeyedState.size()); } extractIntersectingState( rawKeyedState, subtaskKeyGroupRange, extractedKeyedStateHandles); } } return extractedKeyedStateHandles; }
private void collectPartionableStates( List<OperatorState> operatorStates, List<List<OperatorStateHandle>> managedOperatorStates, List<List<OperatorStateHandle>> rawOperatorStates) { for (OperatorState operatorState : operatorStates) { List<OperatorStateHandle> managedOperatorState = null; List<OperatorStateHandle> rawOperatorState = null; for (int i = 0; i < operatorState.getParallelism(); i++) { OperatorSubtaskState operatorSubtaskState = operatorState.getState(i); if (operatorSubtaskState != null) { if (managedOperatorState == null) { managedOperatorState = new ArrayList<>(); } managedOperatorState.addAll(operatorSubtaskState.getManagedOperatorState()); if (rawOperatorState == null) { rawOperatorState = new ArrayList<>(); } rawOperatorState.addAll(operatorSubtaskState.getRawOperatorState()); } } managedOperatorStates.add(managedOperatorState); rawOperatorStates.add(rawOperatorState); } }
private Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> reAssignSubKeyedStates( OperatorState operatorState, List<KeyGroupRange> keyGroupPartitions, int subTaskIndex, int newParallelism, int oldParallelism) { Collection<KeyedStateHandle> subManagedKeyedState; Collection<KeyedStateHandle> subRawKeyedState; if (newParallelism == oldParallelism) { if (operatorState.getState(subTaskIndex) != null) { KeyedStateHandle oldSubManagedKeyedState = operatorState.getState(subTaskIndex).getManagedKeyedState(); KeyedStateHandle oldSubRawKeyedState = operatorState.getState(subTaskIndex).getRawKeyedState(); subManagedKeyedState = oldSubManagedKeyedState != null ? Collections.singletonList( oldSubManagedKeyedState) : null; subRawKeyedState = oldSubRawKeyedState != null ? Collections.singletonList( oldSubRawKeyedState) : null; } else { subManagedKeyedState = null; subRawKeyedState = null; } } else { subManagedKeyedState = getManagedKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); subRawKeyedState = getRawKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); } if (subManagedKeyedState == null && subRawKeyedState == null) { return null; } return new Tuple2<>(subManagedKeyedState, subRawKeyedState); }
private Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> reAssignSubKeyedStates( OperatorState operatorState, List<KeyGroupRange> keyGroupPartitions, int subTaskIndex, int newParallelism, int oldParallelism) { List<KeyedStateHandle> subManagedKeyedState; List<KeyedStateHandle> subRawKeyedState; if (newParallelism == oldParallelism) { if (operatorState.getState(subTaskIndex) != null) { subManagedKeyedState = operatorState.getState(subTaskIndex).getManagedKeyedState().asList(); subRawKeyedState = operatorState.getState(subTaskIndex).getRawKeyedState().asList(); } else { subManagedKeyedState = Collections.emptyList(); subRawKeyedState = Collections.emptyList(); } } else { subManagedKeyedState = getManagedKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); subRawKeyedState = getRawKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); } if (subManagedKeyedState.isEmpty() && subRawKeyedState.isEmpty()) { return new Tuple2<>(Collections.emptyList(), Collections.emptyList()); } else { return new Tuple2<>(subManagedKeyedState, subRawKeyedState); } }
private Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> reAssignSubKeyedStates( OperatorState operatorState, List<KeyGroupRange> keyGroupPartitions, int subTaskIndex, int newParallelism, int oldParallelism) { List<KeyedStateHandle> subManagedKeyedState; List<KeyedStateHandle> subRawKeyedState; if (newParallelism == oldParallelism) { if (operatorState.getState(subTaskIndex) != null) { subManagedKeyedState = operatorState.getState(subTaskIndex).getManagedKeyedState().asList(); subRawKeyedState = operatorState.getState(subTaskIndex).getRawKeyedState().asList(); } else { subManagedKeyedState = Collections.emptyList(); subRawKeyedState = Collections.emptyList(); } } else { subManagedKeyedState = getManagedKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); subRawKeyedState = getRawKeyedStateHandles(operatorState, keyGroupPartitions.get(subTaskIndex)); } if (subManagedKeyedState.isEmpty() && subRawKeyedState.isEmpty()) { return new Tuple2<>(Collections.emptyList(), Collections.emptyList()); } else { return new Tuple2<>(subManagedKeyedState, subRawKeyedState); } }
private void collectPartionableStates( List<OperatorState> operatorStates, List<List<OperatorStateHandle>> managedOperatorStates, List<List<OperatorStateHandle>> rawOperatorStates) { for (OperatorState operatorState : operatorStates) { final int parallelism = operatorState.getParallelism(); List<OperatorStateHandle> managedOperatorState = null; List<OperatorStateHandle> rawOperatorState = null; for (int i = 0; i < parallelism; i++) { OperatorSubtaskState operatorSubtaskState = operatorState.getState(i); if (operatorSubtaskState != null) { StateObjectCollection<OperatorStateHandle> managed = operatorSubtaskState.getManagedOperatorState(); StateObjectCollection<OperatorStateHandle> raw = operatorSubtaskState.getRawOperatorState(); if (managedOperatorState == null) { managedOperatorState = new ArrayList<>(parallelism * managed.size()); } managedOperatorState.addAll(managed); if (rawOperatorState == null) { rawOperatorState = new ArrayList<>(parallelism * raw.size()); } rawOperatorState.addAll(raw); } } managedOperatorStates.add(managedOperatorState); rawOperatorStates.add(rawOperatorState); } }
private void collectPartionableStates( List<OperatorState> operatorStates, List<List<OperatorStateHandle>> managedOperatorStates, List<List<OperatorStateHandle>> rawOperatorStates) { for (OperatorState operatorState : operatorStates) { List<OperatorStateHandle> managedOperatorState = null; List<OperatorStateHandle> rawOperatorState = null; for (int i = 0; i < operatorState.getParallelism(); i++) { OperatorSubtaskState operatorSubtaskState = operatorState.getState(i); if (operatorSubtaskState != null) { if (operatorSubtaskState.getManagedOperatorState() != null) { if (managedOperatorState == null) { managedOperatorState = new ArrayList<>(); } managedOperatorState.add(operatorSubtaskState.getManagedOperatorState()); } if (operatorSubtaskState.getRawOperatorState() != null) { if (rawOperatorState == null) { rawOperatorState = new ArrayList<>(); } rawOperatorState.add(operatorSubtaskState.getRawOperatorState()); } } } managedOperatorStates.add(managedOperatorState); rawOperatorStates.add(rawOperatorState); } }
private void collectPartionableStates( List<OperatorState> operatorStates, List<List<OperatorStateHandle>> managedOperatorStates, List<List<OperatorStateHandle>> rawOperatorStates) { for (OperatorState operatorState : operatorStates) { final int parallelism = operatorState.getParallelism(); List<OperatorStateHandle> managedOperatorState = null; List<OperatorStateHandle> rawOperatorState = null; for (int i = 0; i < parallelism; i++) { OperatorSubtaskState operatorSubtaskState = operatorState.getState(i); if (operatorSubtaskState != null) { StateObjectCollection<OperatorStateHandle> managed = operatorSubtaskState.getManagedOperatorState(); StateObjectCollection<OperatorStateHandle> raw = operatorSubtaskState.getRawOperatorState(); if (managedOperatorState == null) { managedOperatorState = new ArrayList<>(parallelism * managed.size()); } managedOperatorState.addAll(managed); if (rawOperatorState == null) { rawOperatorState = new ArrayList<>(parallelism * raw.size()); } rawOperatorState.addAll(raw); } } managedOperatorStates.add(managedOperatorState); rawOperatorStates.add(rawOperatorState); } }