createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, queueFactory); int midpoint = testKeyGroupRange.getStartKeyGroup() + (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2; KeyGroupRange subKeyGroupRange1 = new KeyGroupRange(testKeyGroupRange.getStartKeyGroup(), midpoint); KeyGroupRange subKeyGroupRange2 = new KeyGroupRange(midpoint + 1, testKeyGroupRange.getEndKeyGroup()); .writeTimersSnapshot(new DataOutputViewStreamWrapper(outStream)); if (subKeyGroupRange1.contains(keyGroupIndex)) { snapshot1.put(keyGroupIndex, outStream.toByteArray()); } else if (subKeyGroupRange2.contains(keyGroupIndex)) { snapshot2.put(keyGroupIndex, outStream.toByteArray()); } else {
/** * Tests list serialization utils. */ @Test public void testListSerialization() throws Exception { final long key = 0L; final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0); // objects for heap state list serialisation final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = new HeapKeyedStateBackend<>( mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, ClassLoader.getSystemClassLoader(), keyGroupRange.getNumberOfKeyGroups(), keyGroupRange, async, new ExecutionConfig(), TestLocalRecoveryConfig.disabled(), new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128), TtlTimeProvider.DEFAULT ); longHeapKeyedStateBackend.setCurrentKey(key); final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState( VoidNamespaceSerializer.INSTANCE, new ListStateDescriptor<>("test", LongSerializer.INSTANCE)); testListSerialization(key, listState); }
final byte[] endKeyGroupBytes = new byte[keyGroupPrefixBytes]; if (currentKeyGroupRange.getStartKeyGroup() < targetKeyGroupRange.getStartKeyGroup()) { RocksDBKeySerializationUtils.serializeKeyGroup( currentKeyGroupRange.getStartKeyGroup(), beginKeyGroupBytes); RocksDBKeySerializationUtils.serializeKeyGroup( targetKeyGroupRange.getStartKeyGroup(), endKeyGroupBytes); deleteRange(db, columnFamilyHandles, beginKeyGroupBytes, endKeyGroupBytes); if (currentKeyGroupRange.getEndKeyGroup() > targetKeyGroupRange.getEndKeyGroup()) { RocksDBKeySerializationUtils.serializeKeyGroup( targetKeyGroupRange.getEndKeyGroup() + 1, beginKeyGroupBytes); RocksDBKeySerializationUtils.serializeKeyGroup( currentKeyGroupRange.getEndKeyGroup() + 1, endKeyGroupBytes); deleteRange(db, columnFamilyHandles, beginKeyGroupBytes, endKeyGroupBytes);
/** * Returns a key-group range with offsets which is the intersection of the internal key-group range with the given * key-group range. * * @param keyGroupRange Key-group range to intersect with the internal key-group range. * @return The key-group range with offsets for the intersection of the internal key-group range with the given * key-group range. */ public KeyGroupRangeOffsets getIntersection(KeyGroupRange keyGroupRange) { Preconditions.checkNotNull(keyGroupRange); KeyGroupRange intersection = this.keyGroupRange.getIntersection(keyGroupRange); long[] subOffsets = new long[intersection.getNumberOfKeyGroups()]; if(subOffsets.length > 0) { System.arraycopy( offsets, computeKeyGroupIndex(intersection.getStartKeyGroup()), subOffsets, 0, subOffsets.length); } return new KeyGroupRangeOffsets(intersection, subOffsets); }
private static int getKeyInKeyGroupRange(KeyGroupRange range, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (!range.contains(KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism))) { result = rand.nextInt(); } return result; } }
public InternalTimerServiceImplTest(int startKeyGroup, int endKeyGroup, int maxParallelism) { this.testKeyGroupRange = new KeyGroupRange(startKeyGroup, endKeyGroup); this.maxParallelism = maxParallelism; }
private int globalKeyGroupToLocalIndex(int keyGroup) { checkArgument(keyGroupRange.contains(keyGroup)); return keyGroup - keyGroupRange.getStartKeyGroup(); }
KeyGroupRange subKeyGroupRange1 = new KeyGroupRange(0, (maxParallelism / 2) - 1); KeyGroupRange subKeyGroupRange2 = new KeyGroupRange(subKeyGroupRange1.getEndKeyGroup() + 1, maxParallelism - 1);
private <T> List<Set<T>> partitionElementsByKeyGroup(KeyGroupedInternalPriorityQueue<T> keyGroupedQueue) { List<Set<T>> result = new ArrayList<>(localKeyGroupRange.getNumberOfKeyGroups()); for (int keyGroup : localKeyGroupRange) { result.add(Collections.unmodifiableSet(keyGroupedQueue.getSubsetForKeyGroup(keyGroup))); } return result; }
@Override public KeyedStateHandle getIntersection(KeyGroupRange otherKeyGroupRange) { return this.keyGroupRange.getIntersection(otherKeyGroupRange).getNumberOfKeyGroups() > 0 ? this : null; }
private int computeKeyGroupIndex(int keyGroup) { int idx = keyGroup - keyGroupRange.getStartKeyGroup(); if (idx < 0 || idx >= offsets.length) { throw new IllegalArgumentException("Key group " + keyGroup + " is not in " + keyGroupRange + "."); } return idx; }
@Override public KeyedStateHandle getIntersection(KeyGroupRange keyGroupRange) { return KeyGroupRange.EMPTY_KEY_GROUP_RANGE.equals(this.keyGroupRange.getIntersection(keyGroupRange)) ? null : this; }
/** * Creates a new {@link NestedMapsStateTable} for the given key context and meta info. * * @param keyContext the key context. * @param metaInfo the meta information for this state table. */ public NestedMapsStateTable(InternalKeyContext<K> keyContext, RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo) { super(keyContext, metaInfo); this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup(); @SuppressWarnings("unchecked") Map<N, Map<K, S>>[] state = (Map<N, Map<K, S>>[]) new Map[keyContext.getKeyGroupRange().getNumberOfKeyGroups()]; this.state = state; }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof KeyGroupRangeOffsets)) { return false; } KeyGroupRangeOffsets that = (KeyGroupRangeOffsets) o; if (keyGroupRange != null ? !keyGroupRange.equals(that.keyGroupRange) : that.keyGroupRange != null) { return false; } return Arrays.equals(offsets, that.offsets); }
@Override public int hashCode() { int result = keyGroupRange != null ? keyGroupRange.hashCode() : 0; result = 31 * result + Arrays.hashCode(offsets); return result; }
/** * Creates key-group range with offsets from the given start key-group to end key-group. The order of given offsets * must be aligned with respect to the key-groups in the range. * * @param rangeStart Start key-group of the range (inclusive) * @param rangeEnd End key-group of the range (inclusive) * @param offsets The aligned array of offsets for the given key-groups. */ public KeyGroupRangeOffsets(int rangeStart, int rangeEnd, long[] offsets) { this(KeyGroupRange.of(rangeStart, rangeEnd), offsets); }
public KeyGroupOffsetsIterator() { this.keyGroupIterator = keyGroupRange.iterator(); }
@Override public KeyedStateHandle getIntersection(KeyGroupRange otherKeyGroupRange) { Preconditions.checkNotNull(otherKeyGroupRange); KeyGroupRange intersectGroups = keyGroupRange.getIntersection(otherKeyGroupRange); if (snapshotHandle == null) { return new KeyGroupsStateSnapshot(intersectGroups); } Map<Integer, Tuple2<Long, Integer>> intersectMetaInfos = new HashMap<>(); for (int group : intersectGroups) { Tuple2<Long, Integer> metaInfo = metaInfos.get(group); if (metaInfo != null) { intersectMetaInfos.put(group, metaInfo); } } return new KeyGroupsStateSnapshot( intersectGroups, intersectMetaInfos, snapshotHandle ); }
RocksDBKeySerializationUtils.serializeKeyGroup(stateBackend.getKeyGroupRange().getStartKeyGroup(), startKeyGroupPrefixBytes); RocksDBKeySerializationUtils.serializeKeyGroup(stateBackend.getKeyGroupRange().getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);
/** * Returns a key-group range with offsets which is the intersection of the internal key-group range with the given * key-group range. * * @param keyGroupRange Key-group range to intersect with the internal key-group range. * @return The key-group range with offsets for the intersection of the internal key-group range with the given * key-group range. */ public KeyGroupRangeOffsets getIntersection(KeyGroupRange keyGroupRange) { Preconditions.checkNotNull(keyGroupRange); KeyGroupRange intersection = this.keyGroupRange.getIntersection(keyGroupRange); long[] subOffsets = new long[intersection.getNumberOfKeyGroups()]; if(subOffsets.length > 0) { System.arraycopy( offsets, computeKeyGroupIndex(intersection.getStartKeyGroup()), subOffsets, 0, subOffsets.length); } return new KeyGroupRangeOffsets(intersection, subOffsets); }