@Override public <S extends State> S getPartitionedState(StateDescriptor<S, ?> stateDescriptor) { try { return stateBackend.getPartitionedState(window, windowSerializer, stateDescriptor); } catch (Exception e) { throw new RuntimeException("Error getting state", e); } }
final K key = this.<K>getKeyedStateBackend().getCurrentKey();
TypeSerializer<K> keySerializer = keyedStateBackend.getKeySerializer(); InternalTimeServiceManager<K> keyedTimeServiceHandler = (InternalTimeServiceManager<K>) timeServiceManager; TimerSerializer<K, N> timerSerializer = new TimerSerializer<>(keySerializer, namespaceSerializer);
@Override public void clear() throws Exception { // TODO we have to collect all keys because otherwise we get ConcurrentModificationExceptions // from flink. We can change this once it's fixed in Flink List<K> keys = backend .getKeys(stateDescriptor.getName(), VoidNamespace.INSTANCE) .collect(Collectors.toList()); for (K key : keys) { backend.setCurrentKey(key); ListState<T> state = backend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor); state.clear(); } }
@Override public void pushBack(T element) throws Exception { ListState<T> state = backend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor); backend.setCurrentKey(keySelector.getKey(element)); state.add(element); }
@Override public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) { // We need to decode the key final ByteBuffer encodedKey = (ByteBuffer) timer.getKey(); @SuppressWarnings("ByteBufferBackingArray") byte[] bytes = encodedKey.array(); final Object decodedKey; try { decodedKey = CoderUtils.decodeFromByteArray(keyCoder, bytes); } catch (CoderException e) { throw new RuntimeException( String.format(Locale.ENGLISH, "Failed to decode encoded key: %s", Arrays.toString(bytes)), e); } // Prepare the SdkHarnessRunner with the key for the timer sdkHarnessRunner.setCurrentTimerKey(decodedKey); // We have to synchronize to ensure the state backend is not concurrently accessed by the state requests try { stateBackendLock.lock(); getKeyedStateBackend().setCurrentKey(encodedKey); super.fireTimer(timer); } finally { stateBackendLock.unlock(); } }
@Override public void processElement( StreamRecord<WindowedValue<ValueWithRecordId<T>>> streamRecord) throws Exception { ByteBuffer currentKey = keyedStateBackend.getCurrentKey(); int groupIndex = keyedStateBackend.getCurrentKeyGroupIndex(); if (shouldOutput(groupIndex, currentKey)) { WindowedValue<ValueWithRecordId<T>> value = streamRecord.getValue(); output.collect(streamRecord.replace(value.withValue(value.getValue().getValue()))); } }
FlinkKeyGroupBagState(StateTag<BagState<T>> address, StateNamespace namespace, Coder<T> coder) { super( address.getId(), namespace.stringKey(), ListCoder.of(coder), new KeyGroupBagCombiner<>()); this.namespace = namespace; this.address = address; }
@Override public void initializeState(StateInitializationContext context) throws Exception { if (getKeyedStateBackend() != null) { int totalKeyGroups = getKeyedStateBackend().getNumberOfKeyGroups(); KeyGroupsList localKeyGroupRange = getKeyedStateBackend().getKeyGroupRange(); for (KeyGroupStatePartitionStreamProvider streamProvider : context.getRawKeyedStateInputs()) { DataInputViewStreamWrapper div = new DataInputViewStreamWrapper(streamProvider.getStream()); int keyGroupIdx = streamProvider.getKeyGroupId(); checkArgument(localKeyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range."); // if (this instanceof KeyGroupRestoringOperator) restoreKeyGroupState(keyGroupIdx, div); // We just initialize our timerService if (keyCoder != null) { if (timerService == null) { timerService = new HeapInternalTimerService<>( totalKeyGroups, localKeyGroupRange, this, getRuntimeContext().getProcessingTimeService()); } timerService.restoreTimersForKeyGroup(div, keyGroupIdx, getUserCodeClassloader()); } } } }
private void migrateOldState() throws Exception { getKeyedStateBackend().applyToAllKeys( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>( "nfaOperatorStateName", new NFA.NFASerializer<>(inputSerializer) ), new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() { @Override public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception { MigratedNFA<IN> oldState = state.value(); computationStates.update(new NFAState(oldState.getComputationStates())); org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer(); partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages()); state.clear(); } } ); }
@Override public void clear() throws Exception { // TODO we have to collect all keys because otherwise we get ConcurrentModificationExceptions // from flink. We can change this once it's fixed in Flink List<K> keys = backend .getKeys(stateDescriptor.getName(), VoidNamespace.INSTANCE) .collect(Collectors.toList()); for (K key : keys) { backend.setCurrentKey(key); ListState<T> state = backend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor); state.clear(); } }
@Override public void pushBack(T element) throws Exception { ListState<T> state = backend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor); backend.setCurrentKey(keySelector.getKey(element)); state.add(element); }
private void setTimer(WindowedValue<InputT> timerElement, TimerInternals.TimerData timerData) { try { Object key = keySelector.getKey(timerElement); sdkHarnessRunner.setCurrentTimerKey(key); // We have to synchronize to ensure the state backend is not concurrently accessed by the state requests try { stateBackendLock.lock(); getKeyedStateBackend().setCurrentKey(key); timerInternals.setTimer(timerData); } finally { stateBackendLock.unlock(); } } catch (Exception e) { throw new RuntimeException("Couldn't set timer", e); } finally { sdkHarnessRunner.setCurrentTimerKey(null); } }
FlinkKeyGroupBagState( StateTag<BagState<T>> address, StateNamespace namespace, Coder<T> coder) { super(address.getId(), namespace.stringKey(), ListCoder.of(coder), new KeyGroupBagCombiner<T>()); this.namespace = namespace; this.address = address; }
getKeyedStateBackend().getNumberOfKeyGroups(), getKeyedStateBackend().getKeyGroupRange(), this, getRuntimeContext().getProcessingTimeService());
private void migrateOldState() throws Exception { getKeyedStateBackend().applyToAllKeys( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>( "nfaOperatorStateName", new NFA.NFASerializer<>(inputSerializer) ), new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() { @Override public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception { MigratedNFA<IN> oldState = state.value(); computationStates.update(new NFAState(oldState.getComputationStates())); org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer(); partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages()); state.clear(); } } ); }
@Override public void open() throws Exception { super.open(); this.state = getKeyedStateBackend().getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor); }
@Override public Stream<T> getElements() { return backend .getKeys(stateDescriptor.getName(), VoidNamespace.INSTANCE) .flatMap( key -> { try { backend.setCurrentKey(key); ListState<T> state = backend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor); return StreamSupport.stream(state.get().spliterator(), false); } catch (Exception e) { throw new RuntimeException("Error reading keyed state.", e); } }); }
final K key = this.<K>getKeyedStateBackend().getCurrentKey();
BagState<String> valueForNamespace0 = allState.state(NAMESPACE_1, STRING_BAG_ADDR); BagState<String> valueForNamespace1 = allState.state(NAMESPACE_2, STRING_BAG_ADDR); keyedStateBackend.setCurrentKey(key0); valueForNamespace0.add("0"); valueForNamespace1.add("2"); keyedStateBackend.setCurrentKey(key1); valueForNamespace0.add("1"); valueForNamespace1.add("3");