@Test public void testRestoreAfterScaleUp() throws Exception { OperatorID headOperatorID = new OperatorID(42L, 42L); OperatorID tailOperatorID = new OperatorID(44L, 44L); JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain( headOperatorID, new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.empty()); TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot(); assertEquals(2, stateHandles.getSubtaskStateMappings().size()); // test empty state in case of scale up OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom( new OperatorInstanceID(0, headOperatorID), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState); createRunAndCheckpointOperatorChain( headOperatorID, new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.of(restore)); assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS); }
public static Map<OperatorInstanceID, List<OperatorStateHandle>> applyRepartitioner( OperatorID operatorID, OperatorStateRepartitioner opStateRepartitioner, List<OperatorStateHandle> chainOpParallelStates, int oldParallelism, int newParallelism) { List<List<OperatorStateHandle>> states = applyRepartitioner( opStateRepartitioner, chainOpParallelStates, oldParallelism, newParallelism); Map<OperatorInstanceID, List<OperatorStateHandle>> result = new HashMap<>(states.size()); for (int subtaskIndex = 0; subtaskIndex < states.size(); subtaskIndex++) { checkNotNull(states.get(subtaskIndex) != null, "states.get(subtaskIndex) is null"); result.put(OperatorInstanceID.of(subtaskIndex, operatorID), states.get(subtaskIndex)); } return result; }
public static Map<OperatorInstanceID, List<OperatorStateHandle>> applyRepartitioner( OperatorID operatorID, OperatorStateRepartitioner opStateRepartitioner, List<OperatorStateHandle> chainOpParallelStates, int oldParallelism, int newParallelism) { Map<OperatorInstanceID, List<OperatorStateHandle>> result = new HashMap<>(); List<Collection<OperatorStateHandle>> states = applyRepartitioner( opStateRepartitioner, chainOpParallelStates, oldParallelism, newParallelism); for (int subtaskIndex = 0; subtaskIndex < states.size(); subtaskIndex++) { checkNotNull(states.get(subtaskIndex) != null, "states.get(subtaskIndex) is null"); result .computeIfAbsent(OperatorInstanceID.of(subtaskIndex, operatorID), key -> new ArrayList<>()) .addAll(states.get(subtaskIndex)); } return result; }
public static Map<OperatorInstanceID, List<OperatorStateHandle>> applyRepartitioner( OperatorID operatorID, OperatorStateRepartitioner opStateRepartitioner, List<OperatorStateHandle> chainOpParallelStates, int oldParallelism, int newParallelism) { List<List<OperatorStateHandle>> states = applyRepartitioner( opStateRepartitioner, chainOpParallelStates, oldParallelism, newParallelism); Map<OperatorInstanceID, List<OperatorStateHandle>> result = new HashMap<>(states.size()); for (int subtaskIndex = 0; subtaskIndex < states.size(); subtaskIndex++) { checkNotNull(states.get(subtaskIndex) != null, "states.get(subtaskIndex) is null"); result.put(OperatorInstanceID.of(subtaskIndex, operatorID), states.get(subtaskIndex)); } return result; }
public static OperatorInstanceID of(int subtaskId, OperatorID operatorID) { return new OperatorInstanceID(subtaskId, operatorID); }
private void reDistributeKeyedStates( List<OperatorState> oldOperatorStates, int newParallelism, List<OperatorID> newOperatorIDs, List<KeyGroupRange> newKeyGroupPartitions, Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState) { //TODO: rewrite this method to only use OperatorID checkState(newOperatorIDs.size() == oldOperatorStates.size(), "This method still depends on the order of the new and old operators"); for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) { OperatorState operatorState = oldOperatorStates.get(operatorIndex); int oldParallelism = operatorState.getParallelism(); for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) { OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex)); Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> subKeyedStates = reAssignSubKeyedStates( operatorState, newKeyGroupPartitions, subTaskIndex, newParallelism, oldParallelism); newManagedKeyedState.put(instanceID, subKeyedStates.f0); newRawKeyedState.put(instanceID, subKeyedStates.f1); } } }
public static OperatorInstanceID of(int subtaskId, OperatorID operatorID) { return new OperatorInstanceID(subtaskId, operatorID); }
private void reDistributeKeyedStates( List<OperatorState> oldOperatorStates, int newParallelism, List<OperatorID> newOperatorIDs, List<KeyGroupRange> newKeyGroupPartitions, Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState) { //TODO: rewrite this method to only use OperatorID checkState(newOperatorIDs.size() == oldOperatorStates.size(), "This method still depends on the order of the new and old operators"); for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) { OperatorState operatorState = oldOperatorStates.get(operatorIndex); int oldParallelism = operatorState.getParallelism(); for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) { OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex)); Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> subKeyedStates = reAssignSubKeyedStates( operatorState, newKeyGroupPartitions, subTaskIndex, newParallelism, oldParallelism); newManagedKeyedState.put(instanceID, subKeyedStates.f0); newRawKeyedState.put(instanceID, subKeyedStates.f1); } } }
public static OperatorInstanceID of(int subtaskId, OperatorID operatorID) { return new OperatorInstanceID(subtaskId, operatorID); }
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex)); if (isHeadOperator(operatorIndex, newOperatorIDs)) { Tuple2<Collection<KeyedStateHandle>, Collection<KeyedStateHandle>> subKeyedStates = reAssignSubKeyedStates(
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, operatorID);
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, operatorID);
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, operatorID);