private static int getKeyInKeyGroup(int keyGroup, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism) != keyGroup) { result = rand.nextInt(); } return result; }
final KeyGroupRange keyGroupRange = KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex( taskInfo.getMaxNumberOfParallelSubtasks(), taskInfo.getNumberOfParallelSubtasks(),
@Override public void configure(int maxParallelism) { KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); this.maxParallelism = maxParallelism; } }
int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(key, maxParallelism); expectedResult.add(Tuple2.of(KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, parallelism, keyGroupIndex), numberElements * key)); int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(key, maxParallelism); expectedResult2.add(Tuple2.of(KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, parallelism2, keyGroupIndex), key * (numberElements + numberElements2)));
private CompletableFuture<KvStateResponse> getState( final KvStateRequest request, final boolean forceUpdate) { return getKvStateLookupInfo(request.getJobId(), request.getStateName(), forceUpdate) .thenComposeAsync((Function<KvStateLocation, CompletableFuture<KvStateResponse>>) location -> { final int keyGroupIndex = KeyGroupRangeAssignment.computeKeyGroupForKeyHash( request.getKeyHashCode(), location.getNumKeyGroups()); final InetSocketAddress serverAddress = location.getKvStateServerAddress(keyGroupIndex); if (serverAddress == null) { return FutureUtils.completedExceptionally(new UnknownKvStateKeyGroupLocationException(getServerName())); } else { // Query server final KvStateID kvStateId = location.getKvStateID(keyGroupIndex); final KvStateInternalRequest internalRequest = new KvStateInternalRequest( kvStateId, request.getSerializedKeyAndNamespace()); return kvStateClient.sendRequest(serverAddress, internalRequest); } }, queryExecutor); }
public Integer getOperatorIndex(int maxParallelism, int parallelism) throws Exception { return KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, parallelism, getKeyGroup(maxParallelism)); }
@Override public int selectChannel(SerializationDelegate<StreamRecord<T>> record) { K key; try { key = keySelector.getKey(record.getInstance().getValue()); } catch (Exception e) { throw new RuntimeException("Could not extract key from " + record.getInstance().getValue(), e); } return KeyGroupRangeAssignment.assignKeyToParallelOperator(key, maxParallelism, numberOfChannels); }
configuredMaxParallelism : KeyGroupRangeAssignment.computeDefaultMaxParallelism(parallelism));
int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(key, maxParallelism); expectedResult.add(Tuple2.of(KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, parallelism, keyGroupIndex), numberElements * key)); int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(key, maxParallelism); expectedResult2.add(Tuple2.of(KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, parallelism2, keyGroupIndex), key * (numberElements + numberElements2)));
/** * Assigns the given key to a key-group index. * * @param key the key to assign * @param maxParallelism the maximum supported parallelism, aka the number of key-groups. * @return the key-group to which the given key is assigned */ public static int assignToKeyGroup(Object key, int maxParallelism) { return computeKeyGroupForKeyHash(key.hashCode(), maxParallelism); }
@Override public int[] selectChannels( SerializationDelegate<StreamRecord<T>> record, int numberOfOutputChannels) { K key; try { key = keySelector.getKey(record.getInstance().getValue()); } catch (Exception e) { throw new RuntimeException("Could not extract key from " + record.getInstance().getValue(), e); } returnArray[0] = KeyGroupRangeAssignment.assignKeyToParallelOperator(key, maxParallelism, numberOfOutputChannels); return returnArray; }
configuredMaxParallelism : KeyGroupRangeAssignment.computeDefaultMaxParallelism(numTaskVertices));
private static int getKeyInKeyGroupRange(KeyGroupRange range, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (!range.contains(KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism))) { result = rand.nextInt(); } return result; } }
int keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent1), maxParallelism); assertEquals(1, keygroup); assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 3, keygroup)); assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 2, keygroup)); Event endEvent2 = new Event(45, "end", 1.0); keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent2), maxParallelism); assertEquals(6, keygroup); assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 3, keygroup)); assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 2, keygroup)); Event endEvent3 = new Event(90, "end", 1.0); keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent3), maxParallelism); assertEquals(2, keygroup); assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 3, keygroup)); assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 2, keygroup)); Event endEvent4 = new Event(10, "end", 1.0); keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent4), maxParallelism); assertEquals(9, keygroup); assertEquals(2, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 3, keygroup)); assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 2, keygroup));
@Override public void configure(int maxParallelism) { KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); this.maxParallelism = maxParallelism; } }
/** * Groups the available set of key groups into key group partitions. A key group partition is * the set of key groups which is assigned to the same task. Each set of the returned list * constitutes a key group partition. * <p> * <b>IMPORTANT</b>: The assignment of key groups to partitions has to be in sync with the * KeyGroupStreamPartitioner. * * @param numberKeyGroups Number of available key groups (indexed from 0 to numberKeyGroups - 1) * @param parallelism Parallelism to generate the key group partitioning for * @return List of key group partitions */ public static List<KeyGroupRange> createKeyGroupPartitions(int numberKeyGroups, int parallelism) { Preconditions.checkArgument(numberKeyGroups >= parallelism); List<KeyGroupRange> result = new ArrayList<>(parallelism); for (int i = 0; i < parallelism; ++i) { result.add(KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(numberKeyGroups, parallelism, i)); } return result; }
/** * Assigns the given key to a key-group index. * * @param key the key to assign * @param maxParallelism the maximum supported parallelism, aka the number of key-groups. * @return the key-group to which the given key is assigned */ public static int assignToKeyGroup(Object key, int maxParallelism) { return computeKeyGroupForKeyHash(key.hashCode(), maxParallelism); }
@Override public int[] selectChannels( SerializationDelegate<StreamRecord<T>> record, int numberOfOutputChannels) { K key; try { key = keySelector.getKey(record.getInstance().getValue()); } catch (Exception e) { throw new RuntimeException("Could not extract key from " + record.getInstance().getValue(), e); } returnArray[0] = KeyGroupRangeAssignment.assignKeyToParallelOperator(key, maxParallelism, numberOfOutputChannels); return returnArray; }
configuredMaxParallelism : KeyGroupRangeAssignment.computeDefaultMaxParallelism(numTaskVertices));
private static int getKeyInKeyGroupRange(KeyGroupRange range, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (!range.contains(KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism))) { result = rand.nextInt(); } return result; }