private <T> List<Set<T>> partitionElementsByKeyGroup(KeyGroupedInternalPriorityQueue<T> keyGroupedQueue) { List<Set<T>> result = new ArrayList<>(localKeyGroupRange.getNumberOfKeyGroups()); for (int keyGroup : localKeyGroupRange) { result.add(Collections.unmodifiableSet(keyGroupedQueue.getSubsetForKeyGroup(keyGroup))); } return result; }
@Override public boolean hasNext() { if (null != currentStateHandle && currentOffsetsIterator.hasNext()) { return true; } closeCurrentStream(); while (stateHandleIterator.hasNext()) { currentStateHandle = stateHandleIterator.next(); if (currentStateHandle.getKeyGroupRange().getNumberOfKeyGroups() > 0) { currentOffsetsIterator = currentStateHandle.getGroupRangeOffsets().iterator(); return true; } } return false; }
/** * Tests list serialization utils. */ @Test public void testListSerialization() throws Exception { final long key = 0L; final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0); // objects for heap state list serialisation final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = new HeapKeyedStateBackend<>( mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, ClassLoader.getSystemClassLoader(), keyGroupRange.getNumberOfKeyGroups(), keyGroupRange, async, new ExecutionConfig(), TestLocalRecoveryConfig.disabled(), new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128), TtlTimeProvider.DEFAULT ); longHeapKeyedStateBackend.setCurrentKey(key); final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState( VoidNamespaceSerializer.INSTANCE, new ListStateDescriptor<>("test", LongSerializer.INSTANCE)); testListSerialization(key, listState); }
/** * Tests map serialization utils. */ @Test public void testMapSerialization() throws Exception { final long key = 0L; final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0); // objects for heap state list serialisation final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = new HeapKeyedStateBackend<>( mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, ClassLoader.getSystemClassLoader(), keyGroupRange.getNumberOfKeyGroups(), keyGroupRange, async, new ExecutionConfig(), TestLocalRecoveryConfig.disabled(), new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128), TtlTimeProvider.DEFAULT ); longHeapKeyedStateBackend.setCurrentKey(key); final InternalMapState<Long, VoidNamespace, Long, String> mapState = (InternalMapState<Long, VoidNamespace, Long, String>) longHeapKeyedStateBackend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE)); testMapSerialization(key, mapState); }
/** * Creates key-group range with offsets for the given key-group range, where all offsets are initially zero. * * @param keyGroupRange The range of key-groups. */ public KeyGroupRangeOffsets(KeyGroupRange keyGroupRange) { this(keyGroupRange, new long[keyGroupRange.getNumberOfKeyGroups()]); }
/** * Creates key-group range with offsets for the given key-group range, where all offsets are initially zero. * * @param keyGroupRange The range of key-groups. */ public KeyGroupRangeOffsets(KeyGroupRange keyGroupRange) { this(keyGroupRange, new long[keyGroupRange.getNumberOfKeyGroups()]); }
/** * Creates key-group range with offsets for the given key-group range, where all offsets are initially zero. * * @param keyGroupRange The range of key-groups. */ public KeyGroupRangeOffsets(KeyGroupRange keyGroupRange) { this(keyGroupRange, new long[keyGroupRange.getNumberOfKeyGroups()]); }
/** * Creates key-group range with offsets for the given key-group range, where all offsets are initially zero. * * @param keyGroupRange The range of key-groups. */ public KeyGroupRangeOffsets(KeyGroupRange keyGroupRange) { this(keyGroupRange, new long[keyGroupRange.getNumberOfKeyGroups()]); }
@Override public boolean hasNext() { return iteratorPos < getNumberOfKeyGroups(); }
@Override public int getKeyGroupId(int idx) { if (idx < 0 || idx > getNumberOfKeyGroups()) { throw new IndexOutOfBoundsException("Key group index out of bounds: " + idx); } return startKeyGroup + idx; }
@Override public int getKeyGroupId(int idx) { if (idx < 0 || idx > getNumberOfKeyGroups()) { throw new IndexOutOfBoundsException("Key group index out of bounds: " + idx); } return startKeyGroup + idx; }
@Override public int getKeyGroupId(int idx) { if (idx < 0 || idx > getNumberOfKeyGroups()) { throw new IndexOutOfBoundsException("Key group index out of bounds: " + idx); } return startKeyGroup + idx; }
@Override public int getKeyGroupId(int idx) { if (idx < 0 || idx > getNumberOfKeyGroups()) { throw new IndexOutOfBoundsException("Key group index out of bounds: " + idx); } return startKeyGroup + idx; }
private <T> List<Set<T>> partitionElementsByKeyGroup(KeyGroupedInternalPriorityQueue<T> keyGroupedQueue) { List<Set<T>> result = new ArrayList<>(localKeyGroupRange.getNumberOfKeyGroups()); for (int keyGroup : localKeyGroupRange) { result.add(Collections.unmodifiableSet(keyGroupedQueue.getSubsetForKeyGroup(keyGroup))); } return result; }
@Override public KeyedStateHandle getIntersection(KeyGroupRange otherKeyGroupRange) { return this.keyGroupRange.getIntersection(otherKeyGroupRange).getNumberOfKeyGroups() > 0 ? this : null; }
public KeyedStateCheckpointOutputStream( CheckpointStreamFactory.CheckpointStateOutputStream delegate, KeyGroupRange keyGroupRange) { super(delegate); Preconditions.checkNotNull(keyGroupRange); Preconditions.checkArgument(keyGroupRange != KeyGroupRange.EMPTY_KEY_GROUP_RANGE); this.currentKeyGroup = NO_CURRENT_KEY_GROUP; long[] emptyOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; // mark offsets as currently not set Arrays.fill(emptyOffsets, NO_OFFSET_SET); this.keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, emptyOffsets); }
/** * Creates key-group range with offsets from the given key-group range. The order of given offsets must be aligned * with respect to the key-groups in the range. * * @param keyGroupRange The range of key-groups. * @param offsets The aligned array of offsets for the given key-groups. */ public KeyGroupRangeOffsets(KeyGroupRange keyGroupRange, long[] offsets) { this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange); this.offsets = Preconditions.checkNotNull(offsets); Preconditions.checkArgument(offsets.length == keyGroupRange.getNumberOfKeyGroups()); }
/** * Creates key-group range with offsets from the given key-group range. The order of given offsets must be aligned * with respect to the key-groups in the range. * * @param keyGroupRange The range of key-groups. * @param offsets The aligned array of offsets for the given key-groups. */ public KeyGroupRangeOffsets(KeyGroupRange keyGroupRange, long[] offsets) { this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange); this.offsets = Preconditions.checkNotNull(offsets); Preconditions.checkArgument(offsets.length == keyGroupRange.getNumberOfKeyGroups()); }
/** * Creates key-group range with offsets from the given key-group range. The order of given offsets must be aligned * with respect to the key-groups in the range. * * @param keyGroupRange The range of key-groups. * @param offsets The aligned array of offsets for the given key-groups. */ public KeyGroupRangeOffsets(KeyGroupRange keyGroupRange, long[] offsets) { this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange); this.offsets = Preconditions.checkNotNull(offsets); Preconditions.checkArgument(offsets.length == keyGroupRange.getNumberOfKeyGroups()); }
/** * Creates key-group range with offsets from the given key-group range. The order of given offsets must be aligned * with respect to the key-groups in the range. * * @param keyGroupRange The range of key-groups. * @param offsets The aligned array of offsets for the given key-groups. */ public KeyGroupRangeOffsets(KeyGroupRange keyGroupRange, long[] offsets) { this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange); this.offsets = Preconditions.checkNotNull(offsets); Preconditions.checkArgument(offsets.length == keyGroupRange.getNumberOfKeyGroups()); }