@Override public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) { return newSerializer.getClass() == serializerSupplier.get().getClass() ? TypeSerializerSchemaCompatibility.compatibleAsIs() : TypeSerializerSchemaCompatibility.incompatible(); }
if (outerCompatibility.isIncompatible()) { return TypeSerializerSchemaCompatibility.incompatible(); resolveCompatibility(newNestedSerializers[i], nestedSnapshots[i]); if (compatibility.isIncompatible()) { return TypeSerializerSchemaCompatibility.incompatible(); if (compatibility.isCompatibleAfterMigration()) { nestedSerializerRequiresMigration = true; return (nestedSerializerRequiresMigration || !outerCompatibility.isCompatibleAsIs()) ? TypeSerializerSchemaCompatibility.compatibleAfterMigration() : TypeSerializerSchemaCompatibility.compatibleAsIs();
private static <T> TypeSerializerSchemaCompatibility<T> avroCompatibilityToFlinkCompatibility(SchemaPairCompatibility compatibility) { switch (compatibility.getType()) { case COMPATIBLE: { // The new serializer would be able to read data persisted with *this* serializer, therefore no migration // is required. return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); } case INCOMPATIBLE: { return TypeSerializerSchemaCompatibility.incompatible(); } case RECURSION_IN_PROGRESS: default: return TypeSerializerSchemaCompatibility.incompatible(); } }
if (compatibility.isIncompatible()) { return TypeSerializerSchemaCompatibility.incompatible(); if (compatibility.isCompatibleAfterMigration()) { nestedSerializerRequiresMigration = true; } else if (compatibility.isCompatibleWithReconfiguredSerializer()) { hasReconfiguredNestedSerializers = true; reconfiguredNestedSerializers[i] = compatibility.getReconfiguredSerializer(); } else if (compatibility.isCompatibleAsIs()) { reconfiguredNestedSerializers[i] = newNestedSerializers[i]; } else { return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); @SuppressWarnings("unchecked") TypeSerializer<T> reconfiguredCompositeSerializer = createOuterSerializerWithNestedSerializers(reconfiguredNestedSerializers); return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(reconfiguredCompositeSerializer); return TypeSerializerSchemaCompatibility.compatibleAsIs();
@Override public TypeSerializerSchemaCompatibility<String> resolveSchemaCompatibility(TypeSerializer<String> newSerializer) { // checks the exact class instead of using instanceof; // this ensures that we get a new serializer, and not a ReconfiguredNestedSerializer or RestoredNestedSerializer if (newSerializer.getClass() == NestedSerializer.class) { switch (targetCompatibility) { case COMPATIBLE_AS_IS: return TypeSerializerSchemaCompatibility.compatibleAsIs(); case COMPATIBLE_AFTER_MIGRATION: return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); case COMPATIBLE_WITH_RECONFIGURED_SERIALIZER: return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer( new ReconfiguredNestedSerializer(targetCompatibility)); case INCOMPATIBLE: return TypeSerializerSchemaCompatibility.incompatible(); default: throw new IllegalStateException("Unexpected target compatibility."); } } throw new IllegalArgumentException("Expected the new serializer to be of class " + NestedSerializer.class); }
public static <T> CompatibilityResult<T> resolveCompatibilityResult( TypeSerializerSnapshot<T> precedingSerializerConfigSnapshot, TypeSerializer<T> newSerializer) { TypeSerializerSchemaCompatibility<T> compatibility = precedingSerializerConfigSnapshot.resolveSchemaCompatibility(newSerializer); // everything except "compatible" maps to "requires migration". // at the entry point of the new-to-old-bridge (in the TypeSerializerConfigSnapshot), we // interpret "requiresMigration" as 'incompatible'. That is a precaution because // serializers could previously not specify the 'incompatible' case. return compatibility.isCompatibleAsIs() ? CompatibilityResult.compatible() : CompatibilityResult.requiresMigration(); } }
@Override protected boolean matchesSafely(TypeSerializerSchemaCompatibility<T> testResultCompatibility) { if (expectedCompatibilty.isCompatibleAsIs()) { return testResultCompatibility.isCompatibleAsIs(); } else if (expectedCompatibilty.isIncompatible()) { return testResultCompatibility.isCompatibleAfterMigration(); } else if (expectedCompatibilty.isIncompatible()) { return testResultCompatibility.isIncompatible(); } else if (expectedCompatibilty.isCompatibleWithReconfiguredSerializer()) { return testResultCompatibility.isCompatibleWithReconfiguredSerializer(); } return false; }
@Override public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) { // if there is no configuration snapshot to check against, // then we can only assume that the new serializer is compatible as is return TypeSerializerSchemaCompatibility.compatibleAsIs(); }
@Override public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) { if (!(newSerializer instanceof AvroSerializer)) { return TypeSerializerSchemaCompatibility.incompatible(); } AvroSerializer<?> newAvroSerializer = (AvroSerializer<?>) newSerializer; return resolveSchemaCompatibility(schema, newAvroSerializer.getAvroSchema()); }
if (compatibilityResult.isIncompatible()) { throw new FlinkRuntimeException( new StateMigrationException("The new priority queue serializer must not be incompatible."));
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer, @Nullable StateSnapshotTransformer<SV> snapshotTransformer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; restoredKvStateMetaInfo.updateSnapshotTransformer(snapshotTransformer); TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (!s.isCompatibleAsIs()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
@Test public void testBridgeCompatibilityCheck() throws Exception { TestSerializerConfigSnapshot snap = new TestSerializerConfigSnapshot(); TestSerializer serCompat = new TestSerializer(true); TypeSerializerSchemaCompatibility<Object> resultCompat = snap.resolveSchemaCompatibility(serCompat); assertTrue(resultCompat.isCompatibleAsIs()); TestSerializer serIncompat = new TestSerializer(false); TypeSerializerSchemaCompatibility<Object> resultIncompat = snap.resolveSchemaCompatibility(serIncompat); assertTrue(resultIncompat.isIncompatible()); }
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (s.isCompatibleAfterMigration() || s.isIncompatible()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
@Test public void testCompatibleAfterMigrationPrecedence() throws IOException { final String OUTER_CONFIG = "outer-config"; TypeSerializer<?>[] testNestedSerializers = { new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), new NestedSerializer(TargetCompatibility.COMPATIBLE_AFTER_MIGRATION), new NestedSerializer(TargetCompatibility.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER), new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), }; TypeSerializerSchemaCompatibility<String> compatibility = snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( testNestedSerializers, OUTER_CONFIG, OUTER_CONFIG); Assert.assertTrue(compatibility.isCompatibleAfterMigration()); }
/** * Check that identical enums don't require migration */ @Test public void checkIndenticalEnums() throws Exception { Assert.assertTrue(checkCompatibility(ENUM_A, ENUM_A).isCompatibleAsIs()); }
/** * Resolves writer/reader schema compatibly. * * <p>Checks whenever a new version of a schema (reader) can read values serialized with the old schema (writer). * If the schemas are compatible according to {@code Avro} schema resolution rules * (@see <a href="https://avro.apache.org/docs/current/spec.html#Schema+Resolution">Schema Resolution</a>). */ @VisibleForTesting static <T> TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility( Schema writerSchema, Schema readerSchema) { if (Objects.equals(writerSchema, readerSchema)) { return TypeSerializerSchemaCompatibility.compatibleAsIs(); } final SchemaPairCompatibility compatibility = SchemaCompatibility.checkReaderWriterCompatibility(readerSchema, writerSchema); return avroCompatibilityToFlinkCompatibility(compatibility); }
@Internal TypeSerializerSchemaCompatibility<T> internalResolveSchemaCompatibility( TypeSerializer<T> newSerializer, TypeSerializerSnapshot<?>[] snapshots) { if (newSerializer.getClass() != correspondingSerializerClass) { return TypeSerializerSchemaCompatibility.incompatible(); } S castedNewSerializer = correspondingSerializerClass.cast(newSerializer); // check that outer configuration is compatible; if not, short circuit result if (!isOuterSnapshotCompatible(castedNewSerializer)) { return TypeSerializerSchemaCompatibility.incompatible(); } return constructFinalSchemaCompatibilityResult( getNestedSerializers(castedNewSerializer), snapshots); }
/** * Check that removing enum fields requires migration */ @Test public void checkRemovedField() throws Exception { Assert.assertTrue(checkCompatibility(ENUM_A, ENUM_C).isIncompatible()); }
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer, @Nullable StateSnapshotTransformer<SV> snapshotTransformer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; restoredKvStateMetaInfo.updateSnapshotTransformer(snapshotTransformer); TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (!s.isCompatibleAsIs()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
if (!namespaceCompatibility.isCompatibleAsIs()) { throw new StateMigrationException("For heap backends, the new namespace serializer must be compatible."); restoredKvMetaInfo.updateStateSerializer(newStateSerializer); if (stateCompatibility.isIncompatible()) { throw new StateMigrationException("For heap backends, the new state serializer must not be incompatible.");