@Override protected TypeSerializer<?>[] getNestedSerializers(ListSerializer outerSerializer) { return new TypeSerializer<?>[] { outerSerializer.getElementSerializer() }; } }
/** * Creates a new {@code RocksDBListState}. * * @param columnFamily The RocksDB column family that this state is associated to. * @param namespaceSerializer The serializer for the namespace. * @param valueSerializer The serializer for the state. * @param defaultValue The default value for the state. * @param backend The backend for which this state is bind to. */ private RocksDBListState( ColumnFamilyHandle columnFamily, TypeSerializer<N> namespaceSerializer, TypeSerializer<List<V>> valueSerializer, List<V> defaultValue, RocksDBKeyedStateBackend<K> backend) { super(columnFamily, namespaceSerializer, valueSerializer, defaultValue, backend); ListSerializer<V> castedListSerializer = (ListSerializer<V>) valueSerializer; this.elementSerializer = castedListSerializer.getElementSerializer(); }
/** * Gets the serializer for the elements contained in the list. * * @return The serializer for the elements in the list. */ public TypeSerializer<T> getElementSerializer() { // call getSerializer() here to get the initialization check and proper error message final TypeSerializer<List<T>> rawSerializer = getSerializer(); if (!(rawSerializer instanceof ListSerializer)) { throw new IllegalStateException(); } return ((ListSerializer<T>) rawSerializer).getElementSerializer(); }
@SuppressWarnings("unchecked") static <SV, SEV> StateSnapshotTransformFactory<SV> wrapStateSnapshotTransformFactory( StateDescriptor<?, SV> stateDesc, StateSnapshotTransformFactory<SEV> snapshotTransformFactory, TypeSerializer<SV> stateSerializer) { if (stateDesc instanceof ListStateDescriptor) { TypeSerializer<SEV> elementSerializer = ((ListSerializer<SEV>) stateSerializer).getElementSerializer(); return new RocksDBListStateSnapshotTransformFactory<>(snapshotTransformFactory, elementSerializer); } else if (stateDesc instanceof MapStateDescriptor) { return new RocksDBMapStateSnapshotTransformFactory<>(snapshotTransformFactory); } else { return new RocksDBValueStateSnapshotTransformFactory<>(snapshotTransformFactory); } }
@Override public void migrateSerializedValue( DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<List<V>> priorSerializer, TypeSerializer<List<V>> newSerializer) throws StateMigrationException { Preconditions.checkArgument(priorSerializer instanceof ListSerializer); Preconditions.checkArgument(newSerializer instanceof ListSerializer); TypeSerializer<V> priorElementSerializer = ((ListSerializer<V>) priorSerializer).getElementSerializer(); TypeSerializer<V> newElementSerializer = ((ListSerializer<V>) newSerializer).getElementSerializer(); try { while (serializedOldValueInput.available() > 0) { V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer); newElementSerializer.serialize(element, serializedMigratedValueOutput); if (serializedOldValueInput.available() > 0) { serializedMigratedValueOutput.write(DELIMITER); } } } catch (Exception e) { throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e); } }
public void configure(int maxParallelism, TypeSerializer<?> keySerializer, TypeSerializer<?> valueSerializer) throws Exception { if (valueSerializer instanceof ListSerializer<?>) { super.configure(maxParallelism, keySerializer, ((ListSerializer<?>) valueSerializer).getElementSerializer()); } else { throw new RuntimeException("Doesnt seem to be a list state"); } } }
/** * Gets the serializer for the elements contained in the list. * * @return The serializer for the elements in the list. */ public TypeSerializer<T> getElementSerializer() { // call getSerializer() here to get the initialization check and proper error message final TypeSerializer<List<T>> rawSerializer = getSerializer(); if (!(rawSerializer instanceof ListSerializer)) { throw new IllegalStateException(); } return ((ListSerializer<T>) rawSerializer).getElementSerializer(); }
/** * Returns the serializer for the elements in the state. * * @return The serializer for the elements in the state. */ public TypeSerializer<E> getElementSerializer() { return getValueSerializer().getElementSerializer(); }
/** * Returns the serializer for the elements in the state. * * @return The serializer for the elements in the state. */ public TypeSerializer<E> getElementSerializer() { return getValueSerializer().getElementSerializer(); }
/** * Gets the serializer for the elements contained in the list. * * @return The serializer for the elements in the list. */ public TypeSerializer<T> getElementSerializer() { // call getSerializer() here to get the initialization check and proper error message final TypeSerializer<List<T>> rawSerializer = getSerializer(); if (!(rawSerializer instanceof ListSerializer)) { throw new IllegalStateException(); } return ((ListSerializer<T>) rawSerializer).getElementSerializer(); }
@Override public TypeSerializerSchemaCompatibility<C> resolveSchemaCompatibility(TypeSerializer<C> newSerializer) { if (newSerializer instanceof ListSerializer) { ListSerializerSnapshot<T> listSerializerSnapshot = new ListSerializerSnapshot<>(((ListSerializer<T>) newSerializer).getElementSerializer()); return listSerializerSnapshot.resolveSchemaCompatibility((ListSerializer) newSerializer); } else { return super.resolveSchemaCompatibility(newSerializer); } }
@Override public TypeSerializerSchemaCompatibility<List<T>> resolveSchemaCompatibility(TypeSerializer<List<T>> newSerializer) { checkState(nestedElementSerializerSnapshot != null); if (newSerializer instanceof ListSerializer) { ListSerializer<T> serializer = (ListSerializer<T>) newSerializer; return nestedElementSerializerSnapshot.resolveCompatibilityWithNested( TypeSerializerSchemaCompatibility.compatibleAsIs(), serializer.getElementSerializer()); } else { return TypeSerializerSchemaCompatibility.incompatible(); } }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<V>> safeValueSerializer) throws Exception { Preconditions.checkNotNull(serializedKeyAndNamespace); Preconditions.checkNotNull(safeKeySerializer); Preconditions.checkNotNull(safeNamespaceSerializer); Preconditions.checkNotNull(safeValueSerializer); Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace( serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1); if (result == null) { return null; } final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); // write the same as RocksDB writes lists, with one ',' separator for (int i = 0; i < result.size(); i++) { dupSerializer.serialize(result.get(i), view); if (i < result.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<V>> safeValueSerializer) throws Exception { Preconditions.checkNotNull(serializedKeyAndNamespace); Preconditions.checkNotNull(safeKeySerializer); Preconditions.checkNotNull(safeNamespaceSerializer); Preconditions.checkNotNull(safeValueSerializer); Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace( serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1); if (result == null) { return null; } final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); // write the same as RocksDB writes lists, with one ',' separator for (int i = 0; i < result.size(); i++) { dupSerializer.serialize(result.get(i), view); if (i < result.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); }
@Override public <K, N, V> SubKeyedListState<K, N, V> getSubKeyedListState( ListStateDescriptor<V> descriptor ) throws Exception { if (namespaceSerializer == null) { throw new RuntimeException("The namespace serializer has not been initialized."); } descriptor.initializeSerializerUnlessSet(operator.getExecutionConfig()); return operator.getSubKeyedState(new SubKeyedListStateDescriptor<>( descriptor.getName(), (TypeSerializer<K>) operator.getKeySerializer(), (TypeSerializer<N>) namespaceSerializer, ((ListSerializer<V>) descriptor.getSerializer()).getElementSerializer())); }
@Override public void migrateSerializedValue( DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<List<V>> priorSerializer, TypeSerializer<List<V>> newSerializer) throws StateMigrationException { Preconditions.checkArgument(priorSerializer instanceof ListSerializer); Preconditions.checkArgument(newSerializer instanceof ListSerializer); TypeSerializer<V> priorElementSerializer = ((ListSerializer<V>) priorSerializer).getElementSerializer(); TypeSerializer<V> newElementSerializer = ((ListSerializer<V>) newSerializer).getElementSerializer(); try { while (serializedOldValueInput.available() > 0) { V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer); newElementSerializer.serialize(element, serializedMigratedValueOutput); if (serializedOldValueInput.available() > 0) { serializedMigratedValueOutput.write(DELIMITER); } } } catch (Exception e) { throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e); } }
@Override public void migrateSerializedValue( DataInputDeserializer serializedOldValueInput, DataOutputSerializer serializedMigratedValueOutput, TypeSerializer<List<V>> priorSerializer, TypeSerializer<List<V>> newSerializer) throws StateMigrationException { Preconditions.checkArgument(priorSerializer instanceof ListSerializer); Preconditions.checkArgument(newSerializer instanceof ListSerializer); TypeSerializer<V> priorElementSerializer = ((ListSerializer<V>) priorSerializer).getElementSerializer(); TypeSerializer<V> newElementSerializer = ((ListSerializer<V>) newSerializer).getElementSerializer(); try { while (serializedOldValueInput.available() > 0) { V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer); newElementSerializer.serialize(element, serializedMigratedValueOutput); if (serializedOldValueInput.available() > 0) { serializedMigratedValueOutput.write(DELIMITER); } } } catch (Exception e) { throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e); } }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<List<E>> safeValueSerializer) throws Exception { K key = KvStateSerializer.deserializeKeyAndNamespace(serializedKeyAndNamespace, safeKeySerializer, VoidNamespaceSerializer.INSTANCE).f0; ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); if (stateStorage.lazySerde()) { List<E> value = get(key); if (value == null) { return null; } final TypeSerializer<E> dupSerializer = ((ListSerializer<E>) safeValueSerializer).getElementSerializer(); for (int i = 0; i < value.size(); i++) { dupSerializer.serialize(value.get(i), view); if (i < value.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); } else { return getSerializedValue(key, baos, view, safeKeySerializer); } }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<E>> safeValueSerializer) throws Exception { Tuple2<K, N> tuple = KvStateSerializer.deserializeKeyAndNamespace(serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); K key = tuple.f0; N namespace = tuple.f1; ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); if (stateStorage.lazySerde()) { List<E> value = get(key, namespace); if (value == null) { return null; } final TypeSerializer<E> dupSerializer = ((ListSerializer<E>) safeValueSerializer).getElementSerializer(); for (int i = 0; i < value.size(); i++) { dupSerializer.serialize(value.get(i), view); if (i < value.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); } else { return getSerializedValue(key, namespace, baos, view, safeKeySerializer, safeNamespaceSerializer); } }
public SubKeyedStateDescriptor createSubKeyedStateDescriptor() { Preconditions.checkState(!this.stateType.isKeyedState(), "Expected subKeyed state meta snapshot."); String name = this.getName(); TypeSerializer keySerializer = this.getKeySerializer(); TypeSerializer valueSerializer = this.getValueSerializer(); TypeSerializer namespaceSerializer = this.getNamespaceSerializer(); switch (this.getStateType()) { case SUBKEYED_VALUE: return new SubKeyedValueStateDescriptor(name, keySerializer, namespaceSerializer, valueSerializer); case SUBKEYED_LIST: TypeSerializer elementSerializer = ((ListSerializer) valueSerializer).getElementSerializer(); return new SubKeyedListStateDescriptor(name, keySerializer, namespaceSerializer, elementSerializer); case SUBKEYED_MAP: return new SubKeyedMapStateDescriptor(name, keySerializer, namespaceSerializer, (MapSerializer) valueSerializer); case SUBKEYED_SORTEDMAP: return new SubKeyedSortedMapStateDescriptor(name, keySerializer, namespaceSerializer, (SortedMapSerializer) valueSerializer); } throw new IllegalStateException("Unknown internal state type for " + this.getStateType()); } }