private static int getKeyInKeyGroupRange(KeyGroupRange range, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (!range.contains(KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism))) { result = rand.nextInt(); } return result; } }
private static int getKeyInKeyGroupRange(KeyGroupRange range, int maxParallelism) { Random rand = new Random(System.currentTimeMillis()); int result = rand.nextInt(); while (!range.contains(KeyGroupRangeAssignment.assignToKeyGroup(result, maxParallelism))) { result = rand.nextInt(); } return result; }
/** * Restore the timers (both processing and event time ones) for a given {@code keyGroupIdx}. * * @param restoredSnapshot the restored snapshot containing the key-group's timers, * and the serializers that were used to write them * @param keyGroupIdx the id of the key-group to be put in the snapshot. */ @SuppressWarnings("unchecked") public void restoreTimersForKeyGroup(InternalTimersSnapshot<?, ?> restoredSnapshot, int keyGroupIdx) { this.restoredTimersSnapshot = (InternalTimersSnapshot<K, N>) restoredSnapshot; if (areSnapshotSerializersIncompatible(restoredSnapshot)) { throw new IllegalArgumentException("Tried to restore timers " + "for the same service with different serializers."); } this.keyDeserializer = restoredTimersSnapshot.getKeySerializer(); this.namespaceDeserializer = restoredTimersSnapshot.getNamespaceSerializer(); checkArgument(localKeyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range."); // restore the event time timers eventTimeTimersQueue.addAll(restoredTimersSnapshot.getEventTimeTimers()); // restore the processing time timers processingTimeTimersQueue.addAll(restoredTimersSnapshot.getProcessingTimeTimers()); }
protected <K> InternalTimeServiceManager<K> internalTimeServiceManager( AbstractKeyedStateBackend<K> keyedStatedBackend, KeyContext keyContext, //the operator Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception { if (keyedStatedBackend == null) { return null; } final KeyGroupRange keyGroupRange = keyedStatedBackend.getKeyGroupRange(); final InternalTimeServiceManager<K> timeServiceManager = new InternalTimeServiceManager<>( keyGroupRange, keyContext, keyedStatedBackend, processingTimeService, keyedStatedBackend.requiresLegacySynchronousTimerSnapshots()); // and then initialize the timer services for (KeyGroupStatePartitionStreamProvider streamProvider : rawKeyedStates) { int keyGroupIdx = streamProvider.getKeyGroupId(); Preconditions.checkArgument(keyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range."); timeServiceManager.restoreStateForKeyGroup( streamProvider.getStream(), keyGroupIdx, environment.getUserClassLoader()); } return timeServiceManager; }
Preconditions.checkState(rocksDBKeyedStateBackend.getKeyGroupRange().contains(keyGroup), "The key group must belong to the backend");
private int globalKeyGroupToLocalIndex(int keyGroup) { checkArgument(keyGroupRange.contains(keyGroup)); return keyGroup - keyGroupRange.getStartKeyGroup(); }
private int globalKeyGroupToLocalIndex(int keyGroup) { checkArgument(keyGroupRange.contains(keyGroup)); return keyGroup - keyGroupRange.getStartKeyGroup(); }
/** * Restore the timers (both processing and event time ones) for a given {@code keyGroupIdx}. * * @param restoredSnapshot the restored snapshot containing the key-group's timers, * and the serializers that were used to write them * @param keyGroupIdx the id of the key-group to be put in the snapshot. */ @SuppressWarnings("unchecked") public void restoreTimersForKeyGroup(InternalTimersSnapshot<?, ?> restoredSnapshot, int keyGroupIdx) { this.restoredTimersSnapshot = (InternalTimersSnapshot<K, N>) restoredSnapshot; if (areSnapshotSerializersIncompatible(restoredSnapshot)) { throw new IllegalArgumentException("Tried to restore timers " + "for the same service with different serializers."); } this.keyDeserializer = restoredTimersSnapshot.getKeySerializer(); this.namespaceDeserializer = restoredTimersSnapshot.getNamespaceSerializer(); checkArgument(localKeyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range."); // restore the event time timers eventTimeTimersQueue.addAll(restoredTimersSnapshot.getEventTimeTimers()); // restore the processing time timers processingTimeTimersQueue.addAll(restoredTimersSnapshot.getProcessingTimeTimers()); }
/** * Restore the timers (both processing and event time ones) for a given {@code keyGroupIdx}. * * @param restoredSnapshot the restored snapshot containing the key-group's timers, * and the serializers that were used to write them * @param keyGroupIdx the id of the key-group to be put in the snapshot. */ @SuppressWarnings("unchecked") public void restoreTimersForKeyGroup(InternalTimersSnapshot<?, ?> restoredSnapshot, int keyGroupIdx) { this.restoredTimersSnapshot = (InternalTimersSnapshot<K, N>) restoredSnapshot; if (areSnapshotSerializersIncompatible(restoredSnapshot)) { throw new IllegalArgumentException("Tried to restore timers " + "for the same service with different serializers."); } this.keyDeserializer = restoredTimersSnapshot.getKeySerializer(); this.namespaceDeserializer = restoredTimersSnapshot.getNamespaceSerializer(); checkArgument(localKeyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range."); // restore the event time timers eventTimeTimersQueue.addAll(restoredTimersSnapshot.getEventTimeTimers()); // restore the processing time timers processingTimeTimersQueue.addAll(restoredTimersSnapshot.getProcessingTimeTimers()); }
private void readStateHandleStateData( FSDataInputStream fsDataInputStream, DataInputViewStreamWrapper inView, KeyGroupRangeOffsets keyGroupOffsets, Map<Integer, StateMetaInfoSnapshot> kvStatesById, int numStates, int readVersion, boolean isCompressed) throws IOException { final StreamCompressionDecorator streamCompressionDecorator = isCompressed ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) { int keyGroupIndex = groupOffset.f0; long offset = groupOffset.f1; // Check that restored key groups all belong to the backend. Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend."); fsDataInputStream.seek(offset); int writtenKeyGroupIndex = inView.readInt(); Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex, "Unexpected key-group in restore."); try (InputStream kgCompressionInStream = streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) { readKeyGroupStateData( kgCompressionInStream, kvStatesById, keyGroupIndex, numStates, readVersion); } } }
private void readStateHandleStateData( FSDataInputStream fsDataInputStream, DataInputViewStreamWrapper inView, KeyGroupRangeOffsets keyGroupOffsets, Map<Integer, StateMetaInfoSnapshot> kvStatesById, int numStates, int readVersion, boolean isCompressed) throws IOException { final StreamCompressionDecorator streamCompressionDecorator = isCompressed ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) { int keyGroupIndex = groupOffset.f0; long offset = groupOffset.f1; // Check that restored key groups all belong to the backend. Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend."); fsDataInputStream.seek(offset); int writtenKeyGroupIndex = inView.readInt(); Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex, "Unexpected key-group in restore."); try (InputStream kgCompressionInStream = streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) { readKeyGroupStateData( kgCompressionInStream, kvStatesById, keyGroupIndex, numStates, readVersion); } } }
if (stateBackend.keyGroupRange.contains(keyGroup)) { stateBackend.db.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
protected <K> InternalTimeServiceManager<K> internalTimeServiceManager( AbstractKeyedStateBackend<K> keyedStatedBackend, KeyContext keyContext, //the operator Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception { if (keyedStatedBackend == null) { return null; } final KeyGroupRange keyGroupRange = keyedStatedBackend.getKeyGroupRange(); final InternalTimeServiceManager<K> timeServiceManager = new InternalTimeServiceManager<>( keyGroupRange, keyContext, keyedStatedBackend, processingTimeService, keyedStatedBackend.requiresLegacySynchronousTimerSnapshots()); // and then initialize the timer services for (KeyGroupStatePartitionStreamProvider streamProvider : rawKeyedStates) { int keyGroupIdx = streamProvider.getKeyGroupId(); Preconditions.checkArgument(keyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range."); timeServiceManager.restoreStateForKeyGroup( streamProvider.getStream(), keyGroupIdx, environment.getUserClassLoader()); } return timeServiceManager; }
protected <K> InternalTimeServiceManager<K> internalTimeServiceManager( AbstractKeyedStateBackend<K> keyedStatedBackend, KeyContext keyContext, //the operator Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception { if (keyedStatedBackend == null) { return null; } final KeyGroupRange keyGroupRange = keyedStatedBackend.getKeyGroupRange(); final InternalTimeServiceManager<K> timeServiceManager = new InternalTimeServiceManager<>( keyGroupRange, keyContext, keyedStatedBackend, processingTimeService, keyedStatedBackend.requiresLegacySynchronousTimerSnapshots()); // and then initialize the timer services for (KeyGroupStatePartitionStreamProvider streamProvider : rawKeyedStates) { int keyGroupIdx = streamProvider.getKeyGroupId(); Preconditions.checkArgument(keyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range."); timeServiceManager.restoreStateForKeyGroup( streamProvider.getStream(), keyGroupIdx, environment.getUserClassLoader()); } return timeServiceManager; }
Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");
Preconditions.checkState(rocksDBKeyedStateBackend.getKeyGroupRange().contains(keyGroup), "The key group must belong to the backend");
Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");
Preconditions.checkState(rocksDBKeyedStateBackend.getKeyGroupRange().contains(keyGroup), "The key group must belong to the backend");
Preconditions.checkState(rocksDBKeyedStateBackend.getKeyGroupRange().contains(keyGroup), "The key group must belong to the backend");