private void reDistributePartitionableStates( List<OperatorState> operatorStates, int newParallelism, List<List<Collection<OperatorStateHandle>>> newManagedOperatorStates, List<List<Collection<OperatorStateHandle>>> newRawOperatorStates) { //collect the old partitionalbe state List<List<OperatorStateHandle>> oldManagedOperatorStates = new ArrayList<>(); List<List<OperatorStateHandle>> oldRawOperatorStates = new ArrayList<>(); collectPartionableStates(operatorStates, oldManagedOperatorStates, oldRawOperatorStates); //redistribute OperatorStateRepartitioner opStateRepartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE; for (int operatorIndex = 0; operatorIndex < operatorStates.size(); operatorIndex++) { int oldParallelism = operatorStates.get(operatorIndex).getParallelism(); newManagedOperatorStates.add(applyRepartitioner(opStateRepartitioner, oldManagedOperatorStates.get(operatorIndex), oldParallelism, newParallelism)); newRawOperatorStates.add(applyRepartitioner(opStateRepartitioner, oldRawOperatorStates.get(operatorIndex), oldParallelism, newParallelism)); } }
private void collectPartionableStates( List<OperatorState> operatorStates, List<List<OperatorStateHandle>> managedOperatorStates, List<List<OperatorStateHandle>> rawOperatorStates) { for (OperatorState operatorState : operatorStates) { List<OperatorStateHandle> managedOperatorState = null; List<OperatorStateHandle> rawOperatorState = null; for (int i = 0; i < operatorState.getParallelism(); i++) { OperatorSubtaskState operatorSubtaskState = operatorState.getState(i); if (operatorSubtaskState != null) { if (managedOperatorState == null) { managedOperatorState = new ArrayList<>(); } managedOperatorState.addAll(operatorSubtaskState.getManagedOperatorState()); if (rawOperatorState == null) { rawOperatorState = new ArrayList<>(); } rawOperatorState.addAll(operatorSubtaskState.getRawOperatorState()); } } managedOperatorStates.add(managedOperatorState); rawOperatorStates.add(rawOperatorState); } }
int oldParallelism = oldOperatorStates.get(operatorIndex).getParallelism(); newManagedOperatorStates.putAll(applyRepartitioner( operatorID,
int oldParallelism = oldOperatorStates.get(operatorIndex).getParallelism(); newManagedOperatorStates.putAll(applyRepartitioner( operatorID,
int oldParallelism = oldOperatorStates.get(operatorIndex).getParallelism(); newManagedOperatorStates.putAll(applyRepartitioner( operatorID,
private void reDistributeKeyedStates( List<OperatorState> oldOperatorStates, int newParallelism, List<OperatorID> newOperatorIDs, List<KeyGroupRange> newKeyGroupPartitions, Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState) { //TODO: rewrite this method to only use OperatorID checkState(newOperatorIDs.size() == oldOperatorStates.size(), "This method still depends on the order of the new and old operators"); for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) { OperatorState operatorState = oldOperatorStates.get(operatorIndex); int oldParallelism = operatorState.getParallelism(); for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) { OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex)); Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> subKeyedStates = reAssignSubKeyedStates( operatorState, newKeyGroupPartitions, subTaskIndex, newParallelism, oldParallelism); newManagedKeyedState.put(instanceID, subKeyedStates.f0); newRawKeyedState.put(instanceID, subKeyedStates.f1); } } }
private void reDistributeKeyedStates( List<OperatorState> oldOperatorStates, int newParallelism, List<OperatorID> newOperatorIDs, List<KeyGroupRange> newKeyGroupPartitions, Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState) { //TODO: rewrite this method to only use OperatorID checkState(newOperatorIDs.size() == oldOperatorStates.size(), "This method still depends on the order of the new and old operators"); for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) { OperatorState operatorState = oldOperatorStates.get(operatorIndex); int oldParallelism = operatorState.getParallelism(); for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) { OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex)); Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> subKeyedStates = reAssignSubKeyedStates( operatorState, newKeyGroupPartitions, subTaskIndex, newParallelism, oldParallelism); newManagedKeyedState.put(instanceID, subKeyedStates.f0); newRawKeyedState.put(instanceID, subKeyedStates.f1); } } }
private void collectPartionableStates( List<OperatorState> operatorStates, List<List<OperatorStateHandle>> managedOperatorStates, List<List<OperatorStateHandle>> rawOperatorStates) { for (OperatorState operatorState : operatorStates) { final int parallelism = operatorState.getParallelism(); List<OperatorStateHandle> managedOperatorState = null; List<OperatorStateHandle> rawOperatorState = null; for (int i = 0; i < parallelism; i++) { OperatorSubtaskState operatorSubtaskState = operatorState.getState(i); if (operatorSubtaskState != null) { StateObjectCollection<OperatorStateHandle> managed = operatorSubtaskState.getManagedOperatorState(); StateObjectCollection<OperatorStateHandle> raw = operatorSubtaskState.getRawOperatorState(); if (managedOperatorState == null) { managedOperatorState = new ArrayList<>(parallelism * managed.size()); } managedOperatorState.addAll(managed); if (rawOperatorState == null) { rawOperatorState = new ArrayList<>(parallelism * raw.size()); } rawOperatorState.addAll(raw); } } managedOperatorStates.add(managedOperatorState); rawOperatorStates.add(rawOperatorState); } }
private void collectPartionableStates( List<OperatorState> operatorStates, List<List<OperatorStateHandle>> managedOperatorStates, List<List<OperatorStateHandle>> rawOperatorStates) { for (OperatorState operatorState : operatorStates) { List<OperatorStateHandle> managedOperatorState = null; List<OperatorStateHandle> rawOperatorState = null; for (int i = 0; i < operatorState.getParallelism(); i++) { OperatorSubtaskState operatorSubtaskState = operatorState.getState(i); if (operatorSubtaskState != null) { if (operatorSubtaskState.getManagedOperatorState() != null) { if (managedOperatorState == null) { managedOperatorState = new ArrayList<>(); } managedOperatorState.add(operatorSubtaskState.getManagedOperatorState()); } if (operatorSubtaskState.getRawOperatorState() != null) { if (rawOperatorState == null) { rawOperatorState = new ArrayList<>(); } rawOperatorState.add(operatorSubtaskState.getRawOperatorState()); } } } managedOperatorStates.add(managedOperatorState); rawOperatorStates.add(rawOperatorState); } }
int parallelism = operatorState.getParallelism(); dos.writeInt(parallelism); dos.writeInt(operatorState.getMaxParallelism());
int parallelism = operatorState.getParallelism(); dos.writeInt(parallelism); dos.writeInt(operatorState.getMaxParallelism());
private void collectPartionableStates( List<OperatorState> operatorStates, List<List<OperatorStateHandle>> managedOperatorStates, List<List<OperatorStateHandle>> rawOperatorStates) { for (OperatorState operatorState : operatorStates) { final int parallelism = operatorState.getParallelism(); List<OperatorStateHandle> managedOperatorState = null; List<OperatorStateHandle> rawOperatorState = null; for (int i = 0; i < parallelism; i++) { OperatorSubtaskState operatorSubtaskState = operatorState.getState(i); if (operatorSubtaskState != null) { StateObjectCollection<OperatorStateHandle> managed = operatorSubtaskState.getManagedOperatorState(); StateObjectCollection<OperatorStateHandle> raw = operatorSubtaskState.getRawOperatorState(); if (managedOperatorState == null) { managedOperatorState = new ArrayList<>(parallelism * managed.size()); } managedOperatorState.addAll(managed); if (rawOperatorState == null) { rawOperatorState = new ArrayList<>(parallelism * raw.size()); } rawOperatorState.addAll(raw); } } managedOperatorStates.add(managedOperatorState); rawOperatorStates.add(rawOperatorState); } }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> extractedKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> rawKeyedState = operatorState.getState(i).getRawKeyedState(); if (extractedKeyedStateHandles == null) { extractedKeyedStateHandles = new ArrayList<>(parallelism * rawKeyedState.size()); } extractIntersectingState( rawKeyedState, subtaskKeyGroupRange, extractedKeyedStateHandles); } } return extractedKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> keyedStateHandles = operatorState.getState(i).getManagedKeyedState(); if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(parallelism * keyedStateHandles.size()); } extractIntersectingState( keyedStateHandles, subtaskKeyGroupRange, subtaskKeyedStateHandles); } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = new ArrayList<>(); for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> keyedStateHandles = operatorState.getState(i).getManagedKeyedState(); extractIntersectingState( keyedStateHandles, subtaskKeyGroupRange, subtaskKeyedStateHandles); } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> extractedKeyedStateHandles = new ArrayList<>(); for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> rawKeyedState = operatorState.getState(i).getRawKeyedState(); extractIntersectingState( rawKeyedState, subtaskKeyGroupRange, extractedKeyedStateHandles); } } return extractedKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> keyedStateHandles = operatorState.getState(i).getManagedKeyedState(); if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(parallelism * keyedStateHandles.size()); } extractIntersectingState( keyedStateHandles, subtaskKeyGroupRange, subtaskKeyedStateHandles); } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { final int parallelism = operatorState.getParallelism(); List<KeyedStateHandle> extractedKeyedStateHandles = null; for (int i = 0; i < parallelism; i++) { if (operatorState.getState(i) != null) { Collection<KeyedStateHandle> rawKeyedState = operatorState.getState(i).getRawKeyedState(); if (extractedKeyedStateHandles == null) { extractedKeyedStateHandles = new ArrayList<>(parallelism * rawKeyedState.size()); } extractIntersectingState( rawKeyedState, subtaskKeyGroupRange, extractedKeyedStateHandles); } } return extractedKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle managedKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all managedKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getManagedKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null && operatorState.getState(i).getManagedKeyedState() != null) { KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getManagedKeyedState().getIntersection(subtaskKeyGroupRange); if (intersectedKeyedStateHandle != null) { if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(); } subtaskKeyedStateHandles.add(intersectedKeyedStateHandle); } } } return subtaskKeyedStateHandles; }
/** * Collect {@link KeyGroupsStateHandle rawKeyedStateHandles} which have intersection with given * {@link KeyGroupRange} from {@link TaskState operatorState} * * @param operatorState all state handles of a operator * @param subtaskKeyGroupRange the KeyGroupRange of a subtask * @return all rawKeyedStateHandles which have intersection with given KeyGroupRange */ public static List<KeyedStateHandle> getRawKeyedStateHandles( OperatorState operatorState, KeyGroupRange subtaskKeyGroupRange) { List<KeyedStateHandle> subtaskKeyedStateHandles = null; for (int i = 0; i < operatorState.getParallelism(); i++) { if (operatorState.getState(i) != null && operatorState.getState(i).getRawKeyedState() != null) { KeyedStateHandle intersectedKeyedStateHandle = operatorState.getState(i).getRawKeyedState().getIntersection(subtaskKeyGroupRange); if (intersectedKeyedStateHandle != null) { if (subtaskKeyedStateHandles == null) { subtaskKeyedStateHandles = new ArrayList<>(); } subtaskKeyedStateHandles.add(intersectedKeyedStateHandle); } } } return subtaskKeyedStateHandles; }