/** * Copy-constructor that does not copy transient fields. They will be initialized once required. */ protected KryoSerializer(KryoSerializer<T> toCopy) { this.type = checkNotNull(toCopy.type, "Type class cannot be null."); this.defaultSerializerClasses = toCopy.defaultSerializerClasses; this.defaultSerializers = new LinkedHashMap<>(toCopy.defaultSerializers.size()); this.kryoRegistrations = new LinkedHashMap<>(toCopy.kryoRegistrations.size()); // deep copy the serializer instances in defaultSerializers for (Map.Entry<Class<?>, ExecutionConfig.SerializableSerializer<?>> entry : toCopy.defaultSerializers.entrySet()) { this.defaultSerializers.put(entry.getKey(), deepCopySerializer(entry.getValue())); } // deep copy the serializer instances in kryoRegistrations for (Map.Entry<String, KryoRegistration> entry : toCopy.kryoRegistrations.entrySet()) { KryoRegistration kryoRegistration = entry.getValue(); if (kryoRegistration.getSerializerDefinitionType() == KryoRegistration.SerializerDefinitionType.INSTANCE) { ExecutionConfig.SerializableSerializer<? extends Serializer<?>> serializerInstance = kryoRegistration.getSerializableSerializerInstance(); if (serializerInstance != null) { kryoRegistration = new KryoRegistration( kryoRegistration.getRegisteredClass(), deepCopySerializer(serializerInstance)); } } this.kryoRegistrations.put(entry.getKey(), kryoRegistration); } }
@SuppressWarnings("unchecked") @Override public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot<?> configSnapshot) { if (configSnapshot instanceof KryoSerializerConfigSnapshot) { final KryoSerializerConfigSnapshot<T> config = (KryoSerializerConfigSnapshot<T>) configSnapshot; if (type.equals(config.getTypeClass())) { LinkedHashMap<String, KryoRegistration> reconfiguredRegistrations = config.getKryoRegistrations(); // reconfigure by assuring that classes which were previously registered are registered // again in the exact same order; new class registrations will be appended. // this also overwrites any dummy placeholders that the restored old configuration has reconfiguredRegistrations.putAll(kryoRegistrations); // check if there is still any dummy placeholders even after reconfiguration; // if so, then this new Kryo serializer cannot read old data and is therefore incompatible for (Map.Entry<String, KryoRegistration> reconfiguredRegistrationEntry : reconfiguredRegistrations.entrySet()) { if (reconfiguredRegistrationEntry.getValue().isDummy()) { LOG.warn("The Kryo registration for a previously registered class {} does not have a " + "proper serializer, because its previous serializer cannot be loaded or is no " + "longer valid but a new serializer is not available", reconfiguredRegistrationEntry.getKey()); return CompatibilityResult.requiresMigration(); } } // there's actually no way to tell if new Kryo serializers are compatible with // the previous ones they overwrite; we can only signal compatibility and hope for the best this.kryoRegistrations = reconfiguredRegistrations; return CompatibilityResult.compatible(); } } return CompatibilityResult.requiresMigration(); }
/** * Apply a list of {@link KryoRegistration} to a Kryo instance. The list of registrations is * assumed to already be a final resolution of all possible registration overwrites. * * <p>The registrations are applied in the given order and always specify the registration id as * the next available id in the Kryo instance (providing the id just extra ensures nothing is * overwritten, and isn't strictly required); * * @param kryo the Kryo instance to apply the registrations * @param resolvedRegistrations the registrations, which should already be resolved of all possible registration overwrites */ public static void applyRegistrations(Kryo kryo, Collection<KryoRegistration> resolvedRegistrations) { Serializer<?> serializer; for (KryoRegistration registration : resolvedRegistrations) { serializer = registration.getSerializer(kryo); if (serializer != null) { kryo.register(registration.getRegisteredClass(), serializer, kryo.getNextRegistrationId()); } else { kryo.register(registration.getRegisteredClass(), kryo.getNextRegistrationId()); } } } }
@Override public void write(DataOutputView out) throws IOException { out.writeUTF(kryoRegistration.getRegisteredClass().getName()); final KryoRegistration.SerializerDefinitionType serializerDefinitionType = kryoRegistration.getSerializerDefinitionType(); out.writeInt(serializerDefinitionType.ordinal()); switch (serializerDefinitionType) { case UNSPECIFIED: // nothing else to write break; case CLASS: out.writeUTF(kryoRegistration.getSerializerClass().getName()); break; case INSTANCE: try (final DataOutputViewStream outViewWrapper = new DataOutputViewStream(out)) { InstantiationUtil.serializeObject(outViewWrapper, kryoRegistration.getSerializableSerializerInstance()); } break; default: // this should not happen; adding as a guard for the future throw new IllegalStateException( "Unrecognized Kryo registration serializer definition type: " + serializerDefinitionType); } }
@SuppressWarnings({"rawtypes", "unchecked"}) @Override public void addAvroGenericDataArrayRegistration(LinkedHashMap<String, KryoRegistration> kryoRegistrations) { kryoRegistrations.put(AVRO_GENERIC_DATA_ARRAY, new KryoRegistration(Serializers.DummyAvroRegisteredClass.class, (Class) Serializers.DummyAvroKryoSerializerClass.class)); }
private static LinkedHashMap<String, KryoRegistration> asKryoRegistrations(Class<?> type) { Preconditions.checkNotNull(type); LinkedHashMap<String, KryoRegistration> registration = new LinkedHashMap<>(1); registration.put(type.getClass().getName(), new KryoRegistration(type)); return registration; } }
@Override public void write(DataOutputView out) throws IOException { out.writeUTF(kryoRegistration.getRegisteredClass().getName()); final KryoRegistration.SerializerDefinitionType serializerDefinitionType = kryoRegistration.getSerializerDefinitionType(); out.writeInt(serializerDefinitionType.ordinal()); switch (serializerDefinitionType) { case UNSPECIFIED: // nothing else to write break; case CLASS: out.writeUTF(kryoRegistration.getSerializerClass().getName()); break; case INSTANCE: try (final DataOutputViewStream outViewWrapper = new DataOutputViewStream(out)) { InstantiationUtil.serializeObject(outViewWrapper, kryoRegistration.getSerializableSerializerInstance()); } break; default: // this should not happen; adding as a guard for the future throw new IllegalStateException( "Unrecognized Kryo registration serializer definition type: " + serializerDefinitionType); } }
/** * Copy-constructor that does not copy transient fields. They will be initialized once required. */ protected KryoSerializer(KryoSerializer<T> toCopy) { this.type = checkNotNull(toCopy.type, "Type class cannot be null."); this.defaultSerializerClasses = toCopy.defaultSerializerClasses; this.defaultSerializers = new LinkedHashMap<>(toCopy.defaultSerializers.size()); this.kryoRegistrations = new LinkedHashMap<>(toCopy.kryoRegistrations.size()); // deep copy the serializer instances in defaultSerializers for (Map.Entry<Class<?>, ExecutionConfig.SerializableSerializer<?>> entry : toCopy.defaultSerializers.entrySet()) { this.defaultSerializers.put(entry.getKey(), deepCopySerializer(entry.getValue())); } // deep copy the serializer instances in kryoRegistrations for (Map.Entry<String, KryoRegistration> entry : toCopy.kryoRegistrations.entrySet()) { KryoRegistration kryoRegistration = entry.getValue(); if (kryoRegistration.getSerializerDefinitionType() == KryoRegistration.SerializerDefinitionType.INSTANCE) { ExecutionConfig.SerializableSerializer<? extends Serializer<?>> serializerInstance = kryoRegistration.getSerializableSerializerInstance(); if (serializerInstance != null) { kryoRegistration = new KryoRegistration( kryoRegistration.getRegisteredClass(), deepCopySerializer(serializerInstance)); } } this.kryoRegistrations.put(entry.getKey(), kryoRegistration); } }
kryoRegistrations.put(serializedType.getName(), new KryoRegistration(serializedType)); kryoRegistrations.put(registeredType.getName(), new KryoRegistration(registeredType)); new KryoRegistration( registeredTypeWithSerializerClassEntry.getKey(), registeredTypeWithSerializerClassEntry.getValue())); new KryoRegistration( registeredTypeWithSerializerEntry.getKey(), registeredTypeWithSerializerEntry.getValue()));
@Override public void write(DataOutputView out) throws IOException { out.writeUTF(kryoRegistration.getRegisteredClass().getName()); final KryoRegistration.SerializerDefinitionType serializerDefinitionType = kryoRegistration.getSerializerDefinitionType(); out.writeInt(serializerDefinitionType.ordinal()); switch (serializerDefinitionType) { case UNSPECIFIED: // nothing else to write break; case CLASS: out.writeUTF(kryoRegistration.getSerializerClass().getName()); break; case INSTANCE: try (final DataOutputViewStream outViewWrapper = new DataOutputViewStream(out)) { InstantiationUtil.serializeObject(outViewWrapper, kryoRegistration.getSerializableSerializerInstance()); } break; default: // this should not happen; adding as a guard for the future throw new IllegalStateException( "Unrecognized Kryo registration serializer definition type: " + serializerDefinitionType); } }
/** * Apply a list of {@link KryoRegistration} to a Kryo instance. The list of registrations is * assumed to already be a final resolution of all possible registration overwrites. * * <p>The registrations are applied in the given order and always specify the registration id as * the next available id in the Kryo instance (providing the id just extra ensures nothing is * overwritten, and isn't strictly required); * * @param kryo the Kryo instance to apply the registrations * @param resolvedRegistrations the registrations, which should already be resolved of all possible registration overwrites */ public static void applyRegistrations(Kryo kryo, Collection<KryoRegistration> resolvedRegistrations) { Serializer<?> serializer; for (KryoRegistration registration : resolvedRegistrations) { serializer = registration.getSerializer(kryo); if (serializer != null) { kryo.register(registration.getRegisteredClass(), serializer, kryo.getNextRegistrationId()); } else { kryo.register(registration.getRegisteredClass(), kryo.getNextRegistrationId()); } } } }
@SuppressWarnings("unchecked") @Override public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot<?> configSnapshot) { if (configSnapshot instanceof KryoSerializerConfigSnapshot) { final KryoSerializerConfigSnapshot<T> config = (KryoSerializerConfigSnapshot<T>) configSnapshot; if (type.equals(config.getTypeClass())) { LinkedHashMap<String, KryoRegistration> reconfiguredRegistrations = config.getKryoRegistrations(); // reconfigure by assuring that classes which were previously registered are registered // again in the exact same order; new class registrations will be appended. // this also overwrites any dummy placeholders that the restored old configuration has reconfiguredRegistrations.putAll(kryoRegistrations); // check if there is still any dummy placeholders even after reconfiguration; // if so, then this new Kryo serializer cannot read old data and is therefore incompatible for (Map.Entry<String, KryoRegistration> reconfiguredRegistrationEntry : reconfiguredRegistrations.entrySet()) { if (reconfiguredRegistrationEntry.getValue().isDummy()) { LOG.warn("The Kryo registration for a previously registered class {} does not have a " + "proper serializer, because its previous serializer cannot be loaded or is no " + "longer valid but a new serializer is not available", reconfiguredRegistrationEntry.getKey()); return CompatibilityResult.requiresMigration(); } } // there's actually no way to tell if new Kryo serializers are compatible with // the previous ones they overwrite; we can only signal compatibility and hope for the best this.kryoRegistrations = reconfiguredRegistrations; return CompatibilityResult.compatible(); } } return CompatibilityResult.requiresMigration(); }
/** * Copy-constructor that does not copy transient fields. They will be initialized once required. */ protected KryoSerializer(KryoSerializer<T> toCopy) { this.type = checkNotNull(toCopy.type, "Type class cannot be null."); this.defaultSerializerClasses = toCopy.defaultSerializerClasses; this.defaultSerializers = new LinkedHashMap<>(toCopy.defaultSerializers.size()); this.kryoRegistrations = new LinkedHashMap<>(toCopy.kryoRegistrations.size()); // deep copy the serializer instances in defaultSerializers for (Map.Entry<Class<?>, ExecutionConfig.SerializableSerializer<?>> entry : toCopy.defaultSerializers.entrySet()) { this.defaultSerializers.put(entry.getKey(), deepCopySerializer(entry.getValue())); } // deep copy the serializer instances in kryoRegistrations for (Map.Entry<String, KryoRegistration> entry : toCopy.kryoRegistrations.entrySet()) { KryoRegistration kryoRegistration = entry.getValue(); if (kryoRegistration.getSerializerDefinitionType() == KryoRegistration.SerializerDefinitionType.INSTANCE) { ExecutionConfig.SerializableSerializer<? extends Serializer<?>> serializerInstance = kryoRegistration.getSerializableSerializerInstance(); if (serializerInstance != null) { kryoRegistration = new KryoRegistration( kryoRegistration.getRegisteredClass(), deepCopySerializer(serializerInstance)); } } this.kryoRegistrations.put(entry.getKey(), kryoRegistration); } }
@Override public void addAvroGenericDataArrayRegistration(LinkedHashMap<String, KryoRegistration> kryoRegistrations) { kryoRegistrations.put( GenericData.Array.class.getName(), new KryoRegistration( GenericData.Array.class, new ExecutionConfig.SerializableSerializer<>(new Serializers.SpecificInstanceCollectionSerializerForArrayList()))); }
/** * Apply a list of {@link KryoRegistration} to a Kryo instance. The list of registrations is * assumed to already be a final resolution of all possible registration overwrites. * * <p>The registrations are applied in the given order and always specify the registration id as * the next available id in the Kryo instance (providing the id just extra ensures nothing is * overwritten, and isn't strictly required); * * @param kryo the Kryo instance to apply the registrations * @param resolvedRegistrations the registrations, which should already be resolved of all possible registration overwrites */ public static void applyRegistrations(Kryo kryo, Collection<KryoRegistration> resolvedRegistrations) { Serializer<?> serializer; for (KryoRegistration registration : resolvedRegistrations) { serializer = registration.getSerializer(kryo); if (serializer != null) { kryo.register(registration.getRegisteredClass(), serializer, kryo.getNextRegistrationId()); } else { kryo.register(registration.getRegisteredClass(), kryo.getNextRegistrationId()); } } } }
@SuppressWarnings("unchecked") @Override public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof KryoSerializerConfigSnapshot) { final KryoSerializerConfigSnapshot<T> config = (KryoSerializerConfigSnapshot<T>) configSnapshot; if (type.equals(config.getTypeClass())) { LinkedHashMap<String, KryoRegistration> reconfiguredRegistrations = config.getKryoRegistrations(); // reconfigure by assuring that classes which were previously registered are registered // again in the exact same order; new class registrations will be appended. // this also overwrites any dummy placeholders that the restored old configuration has reconfiguredRegistrations.putAll(kryoRegistrations); // check if there is still any dummy placeholders even after reconfiguration; // if so, then this new Kryo serializer cannot read old data and is therefore incompatible for (Map.Entry<String, KryoRegistration> reconfiguredRegistrationEntry : reconfiguredRegistrations.entrySet()) { if (reconfiguredRegistrationEntry.getValue().isDummy()) { LOG.warn("The Kryo registration for a previously registered class {} does not have a " + "proper serializer, because its previous serializer cannot be loaded or is no " + "longer valid but a new serializer is not available", reconfiguredRegistrationEntry.getKey()); return CompatibilityResult.requiresMigration(); } } // there's actually no way to tell if new Kryo serializers are compatible with // the previous ones they overwrite; we can only signal compatibility and hope for the best this.kryoRegistrations = reconfiguredRegistrations; return CompatibilityResult.compatible(); } } return CompatibilityResult.requiresMigration(); }
kryoRegistration = new KryoRegistration(registeredClass); break; kryoRegistration = new KryoRegistration(registeredClass, serializerClass); break; kryoRegistration = new KryoRegistration(registeredClass, serializerInstance); break;
@SuppressWarnings("unchecked") @Override public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof AvroSerializerConfigSnapshot) { final AvroSerializerConfigSnapshot<T> config = (AvroSerializerConfigSnapshot<T>) configSnapshot; if (type.equals(config.getTypeClass()) && typeToInstantiate.equals(config.getTypeToInstantiate())) { // resolve Kryo registrations; currently, since the Kryo registrations in Avro // are fixed, there shouldn't be a problem with the resolution here. LinkedHashMap<String, KryoRegistration> oldRegistrations = config.getKryoRegistrations(); oldRegistrations.putAll(kryoRegistrations); for (Map.Entry<String, KryoRegistration> reconfiguredRegistrationEntry : kryoRegistrations.entrySet()) { if (reconfiguredRegistrationEntry.getValue().isDummy()) { return CompatibilityResult.requiresMigration(); } } this.kryoRegistrations = oldRegistrations; return CompatibilityResult.compatible(); } } // ends up here if the preceding serializer is not // the ValueSerializer, or serialized data type has changed return CompatibilityResult.requiresMigration(); }
@SuppressWarnings({"rawtypes", "unchecked"}) @Override public void addAvroGenericDataArrayRegistration(LinkedHashMap<String, KryoRegistration> kryoRegistrations) { kryoRegistrations.put(AVRO_GENERIC_DATA_ARRAY, new KryoRegistration(Serializers.DummyAvroRegisteredClass.class, (Class) Serializers.DummyAvroKryoSerializerClass.class)); }
@SuppressWarnings({"rawtypes", "unchecked"}) @Override public void addAvroGenericDataArrayRegistration(LinkedHashMap<String, KryoRegistration> kryoRegistrations) { kryoRegistrations.put(AVRO_GENERIC_DATA_ARRAY, new KryoRegistration(Serializers.DummyAvroRegisteredClass.class, (Class) Serializers.DummyAvroKryoSerializerClass.class)); }