private static int getKeyInKeyGroup(int keyGroup, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism) != keyGroup) { result = rand.nextInt(); } return result; }
private static int getKeyInKeyGroupRange(KeyGroupRange range, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (!range.contains(KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism))) { result = rand.nextInt(); } return result; } }
private static int getKeyInKeyGroupRange(KeyGroupRange range, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (!range.contains(KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism))) { result = rand.nextInt(); } return result; }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<V> safeValueSerializer) throws Exception { //TODO make KvStateSerializer key-group aware to save this round trip and key-group computation Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace( serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(keyAndNamespace.f0, backend.getNumberOfKeyGroups()); RocksDBSerializedCompositeKeyBuilder<K> keyBuilder = new RocksDBSerializedCompositeKeyBuilder<>( safeKeySerializer, backend.getKeyGroupPrefixBytes(), 32 ); keyBuilder.setKeyAndKeyGroup(keyAndNamespace.f0, keyGroup); byte[] key = keyBuilder.buildCompositeKeyNamespace(keyAndNamespace.f1, namespaceSerializer); return backend.db.get(columnFamily, key); }
int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(key, maxParallelism); int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(key, maxParallelism); expectedResult2.add(Tuple2.of(KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, parallelism2, keyGroupIndex), key * (numberElements + numberElements2)));
serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(keyAndNamespace.f0, backend.getNumberOfKeyGroups());
int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(key, maxParallelism); int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(key, maxParallelism); expectedResult2.add(Tuple2.of(KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, parallelism2, keyGroupIndex), key * (numberElements + numberElements2)));
int keyGroupIdx = KeyGroupRangeAssignment.assignToKeyGroup(timer.getKey(), totalNoOfKeyGroups);
int keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent1), maxParallelism); assertEquals(1, keygroup); assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 3, keygroup)); Event endEvent2 = new Event(45, "end", 1.0); keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent2), maxParallelism); assertEquals(6, keygroup); assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 3, keygroup)); Event endEvent3 = new Event(90, "end", 1.0); keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent3), maxParallelism); assertEquals(2, keygroup); assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 3, keygroup)); Event endEvent4 = new Event(10, "end", 1.0); keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent4), maxParallelism); assertEquals(9, keygroup); assertEquals(2, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 3, keygroup));
Event endEvent1 = new Event(7, "end", 1.0); int keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent1), maxParallelism); assertEquals(1, keygroup); assertEquals(0, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 2, keygroup)); Event endEvent2 = new Event(10, "end", 1.0); keygroup = KeyGroupRangeAssignment.assignToKeyGroup(keySelector.getKey(startEvent2), maxParallelism); assertEquals(9, keygroup); assertEquals(1, KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 2, keygroup));
/** * @see KeyedStateBackend */ @Override public void setCurrentKey(K newKey) { this.currentKey = newKey; this.currentKeyGroup = KeyGroupRangeAssignment.assignToKeyGroup(newKey, numberOfKeyGroups); }
@Override public S get(K key, N namespace) { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(key, keyContext.getNumberOfKeyGroups()); return get(key, keyGroup, namespace); }
private int computeKeyGroupIndex(T element) { final Object extractKeyFromElement = keyExtractor.extractKeyFromElement(element); final int keyGroupId = KeyGroupRangeAssignment.assignToKeyGroup(extractKeyFromElement, totalKeyGroups); return globalKeyGroupToLocalIndex(keyGroupId); }
private HashMap<T, T> getDedupMapForElement(T element) { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup( keyExtractor.extractKeyFromElement(element), totalNumberOfKeyGroups); return getDedupMapForKeyGroup(keyGroup); }
@Override public S get(K key, N namespace) { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(key, keyContext.getNumberOfKeyGroups()); return get(key, keyGroup, namespace); }
private int computeKeyGroupIndex(T element) { final Object extractKeyFromElement = keyExtractor.extractKeyFromElement(element); final int keyGroupId = KeyGroupRangeAssignment.assignToKeyGroup(extractKeyFromElement, totalKeyGroups); return globalKeyGroupToLocalIndex(keyGroupId); }
@Override public S get(K key, N namespace) { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(key, keyContext.getNumberOfKeyGroups()); return get(key, keyGroup, namespace); }
/** * This method iterates over the input data and reports the key-group for each element. */ protected void reportAllElementKeyGroups() { Preconditions.checkState(partitioningSource.length >= numberOfElements); for (int i = 0; i < numberOfElements; ++i) { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup( keyExtractorFunction.extractKeyFromElement(partitioningSource[i]), totalKeyGroups); reportKeyGroupOfElementAtIndex(i, keyGroup); } }
/** * Retrieve the set of processing time timers for the key-group this timer belongs to. * * @param timer the timer whose key-group we are searching. * @return the set of registered timers for the key-group. */ private Set<InternalTimer<K, N>> getProcessingTimeTimerSetForTimer(InternalTimer<K, N> timer) { checkArgument(localKeyGroupRange != null, "The operator has not been initialized."); int keyGroupIdx = KeyGroupRangeAssignment.assignToKeyGroup(timer.getKey(), this.totalKeyGroups); return getProcessingTimeTimerSetForKeyGroup(keyGroupIdx); }
/** * This method iterates over the input data and reports the key-group for each element. */ protected void reportAllElementKeyGroups() { Preconditions.checkState(partitioningSource.length >= numberOfElements); for (int i = 0; i < numberOfElements; ++i) { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup( keyExtractorFunction.extractKeyFromElement(partitioningSource[i]), totalKeyGroups); reportKeyGroupOfElementAtIndex(i, keyGroup); } }