cachedMethod.methodClassID = kryo.getRegistration(method.getDeclaringClass()).getId(); cachedMethod.methodIndex = i;
/** * Tests that the registered classes in Kryo did not change. * * <p>Once we have proper serializer versioning this test will become obsolete. * But currently a change in the serializers can break savepoint backwards * compatibility between Flink versions. */ @Test public void testDefaultKryoRegisteredClassesDidNotChange() throws Exception { final Kryo kryo = new KryoSerializer<>(Integer.class, new ExecutionConfig()).getKryo(); try (BufferedReader reader = new BufferedReader(new InputStreamReader( getClass().getClassLoader().getResourceAsStream("flink_11-kryo_registrations")))) { String line; while ((line = reader.readLine()) != null) { String[] split = line.split(","); final int tag = Integer.parseInt(split[0]); final String registeredClass = split[1]; Registration registration = kryo.getRegistration(tag); if (registration == null) { fail(String.format("Registration for %d = %s got lost", tag, registeredClass)); } else if (!registeredClass.equals(registration.getType().getName())) { fail(String.format("Registration for %d = %s changed to %s", tag, registeredClass, registration.getType().getName())); } } } }
Registration registration = kryo.getRegistration(i); String str = registration.getId() + "," + registration.getType().getName(); writer.write(str, 0, str.length());
Registration existing = kryo.getRegistration(id); if (existing != null) { boolean matches = false;
@Test public void testTypeRegistrationFromTypeInfo() { ExecutionConfig conf = new ExecutionConfig(); Serializers.recursivelyRegisterType(new GenericTypeInfo<>(ClassWithNested.class), conf, new HashSet<Class<?>>()); KryoSerializer<String> kryo = new KryoSerializer<>(String.class, conf); // we create Kryo from another type. assertTrue(kryo.getKryo().getRegistration(FromNested.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(ClassWithNested.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(Path.class).getId() > 0); // check if the generic type from one field is also registered (its very likely that // generic types are also used as fields somewhere. assertTrue(kryo.getKryo().getRegistration(FromGeneric1.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(FromGeneric2.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(Node.class).getId() > 0); } }
public void read (Kryo kryo, Input input) { objectID = input.readInt(true); int methodClassID = input.readInt(true); Class methodClass = kryo.getRegistration(methodClassID).getType(); byte methodIndex = input.readByte(); try { cachedMethod = getMethods(kryo, methodClass)[methodIndex]; } catch (IndexOutOfBoundsException ex) { throw new KryoException("Invalid method index " + methodIndex + " for class: " + methodClass.getName()); } Serializer[] serializers = cachedMethod.serializers; Class[] parameterTypes = cachedMethod.method.getParameterTypes(); Object[] args = new Object[serializers.length]; this.args = args; for (int i = 0, n = args.length; i < n; i++) { Serializer serializer = serializers[i]; if (serializer != null) args[i] = kryo.readObjectOrNull(input, parameterTypes[i], serializer); else args[i] = kryo.readClassAndObject(input); } responseData = input.readByte(); } }
@Test public void testTypeRegistration() { ExecutionConfig conf = new ExecutionConfig(); Serializers.recursivelyRegisterType(ClassWithNested.class, conf, new HashSet<Class<?>>()); KryoSerializer<String> kryo = new KryoSerializer<>(String.class, conf); // we create Kryo from another type. Assert.assertTrue(kryo.getKryo().getRegistration(FromNested.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(ClassWithNested.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(Path.class).getId() > 0); // check if the generic type from one field is also registered (its very likely that // generic types are also used as fields somewhere. Assert.assertTrue(kryo.getKryo().getRegistration(FromGeneric1.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(FromGeneric2.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(Node.class).getId() > 0); // register again and make sure classes are still registered ExecutionConfig conf2 = new ExecutionConfig(); Serializers.recursivelyRegisterType(ClassWithNested.class, conf2, new HashSet<Class<?>>()); KryoSerializer<String> kryo2 = new KryoSerializer<>(String.class, conf); assertTrue(kryo2.getKryo().getRegistration(FromNested.class).getId() > 0); }
@Test public void testFoldingStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); @SuppressWarnings("unchecked") FoldFunction<String, TaskInfo> folder = (FoldFunction<String, TaskInfo>) mock(FoldFunction.class); FoldingStateDescriptor<String, TaskInfo> descr = new FoldingStateDescriptor<>("name", null, folder, TaskInfo.class); context.getFoldingState(descr); FoldingStateDescriptor<?, ?> descrIntercepted = (FoldingStateDescriptor<?, ?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testReducingStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); @SuppressWarnings("unchecked") ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class); ReducingStateDescriptor<TaskInfo> descr = new ReducingStateDescriptor<>("name", reducer, TaskInfo.class); context.getReducingState(descr); StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testAggregatingStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); @SuppressWarnings("unchecked") AggregateFunction<String, TaskInfo, String> aggregate = (AggregateFunction<String, TaskInfo, String>) mock(AggregateFunction.class); AggregatingStateDescriptor<String, TaskInfo, String> descr = new AggregatingStateDescriptor<>("name", aggregate, TaskInfo.class); context.getAggregatingState(descr); AggregatingStateDescriptor<?, ?, ?> descrIntercepted = (AggregatingStateDescriptor<?, ?, ?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testValueStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class); context.getState(descr); StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0); }
int testClassId = kryoSerializer.getKryo().getRegistration(TestClass.class).getId(); int testClassAId = kryoSerializer.getKryo().getRegistration(TestClassA.class).getId(); int testClassBId = kryoSerializer.getKryo().getRegistration(TestClassB.class).getId(); kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializer); assertTrue(compatResult.isCompatibleAsIs()); assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId()); assertEquals(testClassAId, kryoSerializer.getKryo().getRegistration(TestClassA.class).getId()); assertEquals(testClassBId, kryoSerializer.getKryo().getRegistration(TestClassB.class).getId());
@Test public void testMapStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); MapStateDescriptor<String, TaskInfo> descr = new MapStateDescriptor<>("name", String.class, TaskInfo.class); context.getMapState(descr); MapStateDescriptor<?, ?> descrIntercepted = (MapStateDescriptor<?, ?>) descriptorCapture.get(); TypeSerializer<?> valueSerializer = descrIntercepted.getValueSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(valueSerializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) valueSerializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testListStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class); context.getListState(descr); ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof ListSerializer); TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer(); assertTrue(elementSerializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testInitializeSerializerAfterSerializationWithCustomConfig() throws Exception { // guard our test assumptions. assertEquals("broken test assumption", -1, new KryoSerializer<>(String.class, new ExecutionConfig()).getKryo() .getRegistration(File.class).getId()); final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(File.class); final TestStateDescriptor<Path> original = new TestStateDescriptor<>("test", Path.class); TestStateDescriptor<Path> clone = CommonTestUtils.createCopySerializable(original); clone.initializeSerializerUnlessSet(config); // serialized one (later initialized) carries the registration assertTrue(((KryoSerializer<?>) clone.getSerializer()).getKryo() .getRegistration(File.class).getId() > 0); }
private void register(Kryo kryo, Registration registration) { int id = registration.getId(); Registration existing = kryo.getRegistration(id); if (existing != null) { throw new IllegalStateException("registration already exists " + existing); } if (this.log.isInfoEnabled()) { this.log.info(String.format("registering %s with serializer %s", registration, registration.getSerializer().getClass().getName())); } kryo.register(registration); }
/** Returns the serializer for the registration for the specified class. * @see #getRegistration(Class) * @see Registration#getSerializer() */ public Serializer getSerializer (Class type) { return getRegistration(type).getSerializer(); }
/** Returns the serializer for the registration for the specified class. * @see #getRegistration(Class) * @see Registration#getSerializer() */ public Serializer getSerializer (Class type) { return getRegistration(type).getSerializer(); }
/** Creates a new instance of a class using {@link Registration#getInstantiator()}. If the registration's instantiator is null, * a new one is set using {@link #newInstantiator(Class)}. */ public <T> T newInstance (Class<T> type) { Registration registration = getRegistration(type); ObjectInstantiator instantiator = registration.getInstantiator(); if (instantiator == null) { instantiator = newInstantiator(type); registration.setInstantiator(instantiator); } return (T)instantiator.newInstance(); }
/** Creates a new instance of a class using {@link Registration#getInstantiator()}. If the registration's instantiator is null, * a new one is set using {@link #newInstantiator(Class)}. */ public <T> T newInstance (Class<T> type) { Registration registration = getRegistration(type); ObjectInstantiator instantiator = registration.getInstantiator(); if (instantiator == null) { instantiator = newInstantiator(type); registration.setInstantiator(instantiator); } return (T)instantiator.newInstance(); }