serializationProxy.getStateMetaInfoSnapshots(); currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());
return serializationProxy.getStateMetaInfoSnapshots(); } finally { if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) {
public static Map<Integer, String> getStateIdMapping(KeyedBackendSerializationProxy<?> proxy) { Map<Integer, String> stateIdMapping = new HashMap<>(); int stateId = 0; for (StateMetaInfoSnapshot snapshot : proxy.getStateMetaInfoSnapshots()) { stateIdMapping.put(stateId, snapshot.getName()); stateId++; } return stateIdMapping; }
@SuppressWarnings("unchecked") public static <T> Optional<TypeSerializer<T>> getSerializer(KeyedBackendSerializationProxy<?> proxy, String stateName) { for (StateMetaInfoSnapshot snapshot : proxy.getStateMetaInfoSnapshots()) { if (snapshot.getName().equals(stateName)) { return Optional .of((TypeSerializer<T>) snapshot .getTypeSerializerSnapshot(CommonSerializerKeys.VALUE_SERIALIZER) .restoreSerializer()); } } return Optional.empty(); }
public OperatorStateWriter(long checkpointId, OperatorState baseOpState, Path newCheckpointBasePath) { this.baseOpState = baseOpState; this.newCheckpointBasePath = newCheckpointBasePath; this.checkpointId = checkpointId; proxy = StateMetadataUtils.getKeyedBackendSerializationProxy(baseOpState).orElse(null); metaSnapshots = new HashMap<>(); if (proxy != null) { proxy.getStateMetaInfoSnapshots() .forEach(ms -> metaSnapshots.put(ms.getName(), new StateMetaInfoSnapshot(ms.getName(), ms.getBackendStateType(), ms.getOptionsImmutable(), ms.getSerializerSnapshotsImmutable(), Maps.transformValues(ms.getSerializerSnapshotsImmutable(), TypeSerializerSnapshot::restoreSerializer)))); } }
public RocksDBCheckpointIterator(IncrementalKeyedStateHandle handle, FilterFunction<String> stateFilter, String localPath) { this.localPath = localPath; this.cancelStreamRegistry = new CloseableRegistry(); List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = StateMetadataUtils .getKeyedBackendSerializationProxy(handle.getMetaStateHandle()).getStateMetaInfoSnapshots(); stateColumnFamilyHandles = new ArrayList<>(stateMetaInfoSnapshots.size() + 1); List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors( stateMetaInfoSnapshots); try { transferAllStateDataToDirectory(handle, new Path(localPath)); this.db = openDB(localPath, stateColumnFamilyDescriptors, stateColumnFamilyHandles); createColumnIterators(stateFilter, stateMetaInfoSnapshots); } catch (Exception e) { throw new IllegalStateException(e); } }
serializationProxy.getStateMetaInfoSnapshots();
serializationProxy.getStateMetaInfoSnapshots(); currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());
serializationProxy.getStateMetaInfoSnapshots();
serializationProxy.getStateMetaInfoSnapshots(); currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());
serializationProxy.getStateMetaInfoSnapshots(); currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());
serializationProxy.getStateMetaInfoSnapshots();
serializationProxy.getStateMetaInfoSnapshots();
/** * Reads Flink's state meta data file from the state handle. */ private List<StateMetaInfoSnapshot> readMetaData( StreamStateHandle metaStateHandle) throws Exception { FSDataInputStream inputStream = null; try { inputStream = metaStateHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerCloseable(inputStream); // isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend, // deserialization of state happens lazily during runtime; we depend on the fact // that the new serializer for states could be compatible, and therefore the restore can continue // without old serializers required to be present. KeyedBackendSerializationProxy<T> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); DataInputView in = new DataInputViewStreamWrapper(inputStream); serializationProxy.read(in); // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (!serializationProxy.getKeySerializerConfigSnapshot() .resolveSchemaCompatibility(stateBackend.keySerializer).isCompatibleAsIs()) { throw new StateMigrationException("The new key serializer must be compatible."); } return serializationProxy.getStateMetaInfoSnapshots(); } finally { if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) { inputStream.close(); } } }
/** * Reads Flink's state meta data file from the state handle. */ private List<StateMetaInfoSnapshot> readMetaData( StreamStateHandle metaStateHandle) throws Exception { FSDataInputStream inputStream = null; try { inputStream = metaStateHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerCloseable(inputStream); // isSerializerPresenceRequired flag is set to false, since for the RocksDB state backend, // deserialization of state happens lazily during runtime; we depend on the fact // that the new serializer for states could be compatible, and therefore the restore can continue // without old serializers required to be present. KeyedBackendSerializationProxy<T> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); DataInputView in = new DataInputViewStreamWrapper(inputStream); serializationProxy.read(in); // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (!serializationProxy.getKeySerializerConfigSnapshot() .resolveSchemaCompatibility(stateBackend.keySerializer).isCompatibleAsIs()) { throw new StateMigrationException("The new key serializer must be compatible."); } return serializationProxy.getStateMetaInfoSnapshots(); } finally { if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) { inputStream.close(); } } }
private List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> readMetaData( StreamStateHandle metaStateHandle) throws Exception { FSDataInputStream inputStream = null; try { inputStream = metaStateHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerClosable(inputStream); KeyedBackendSerializationProxy<T> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); DataInputView in = new DataInputViewStreamWrapper(inputStream); serializationProxy.read(in); // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (CompatibilityUtil.resolveCompatibilityResult( serializationProxy.getKeySerializer(), UnloadableDummyTypeSerializer.class, serializationProxy.getKeySerializerConfigSnapshot(), stateBackend.keySerializer) .isRequiresMigration()) { // TODO replace with state migration; note that key hash codes need to remain the same after migration throw new StateMigrationException("The new key serializer is not compatible to read previous keys. " + "Aborting now since state migration is currently not available"); } return serializationProxy.getStateMetaInfoSnapshots(); } finally { if (inputStream != null) { stateBackend.cancelStreamRegistry.unregisterClosable(inputStream); inputStream.close(); } } }