/** * Tests that the registered classes in Kryo did not change. * * <p>Once we have proper serializer versioning this test will become obsolete. * But currently a change in the serializers can break savepoint backwards * compatibility between Flink versions. */ @Test public void testDefaultKryoRegisteredClassesDidNotChange() throws Exception { final Kryo kryo = new KryoSerializer<>(Integer.class, new ExecutionConfig()).getKryo(); try (BufferedReader reader = new BufferedReader(new InputStreamReader( getClass().getClassLoader().getResourceAsStream("flink_11-kryo_registrations")))) { String line; while ((line = reader.readLine()) != null) { String[] split = line.split(","); final int tag = Integer.parseInt(split[0]); final String registeredClass = split[1]; Registration registration = kryo.getRegistration(tag); if (registration == null) { fail(String.format("Registration for %d = %s got lost", tag, registeredClass)); } else if (!registeredClass.equals(registration.getType().getName())) { fail(String.format("Registration for %d = %s changed to %s", tag, registeredClass, registration.getType().getName())); } } } }
final Kryo kryo = new KryoSerializer<>(Integer.class, new ExecutionConfig()).getKryo(); final int nextId = kryo.getNextRegistrationId();
@Test public void testTypeRegistrationFromTypeInfo() { ExecutionConfig conf = new ExecutionConfig(); Serializers.recursivelyRegisterType(new GenericTypeInfo<>(ClassWithNested.class), conf, new HashSet<Class<?>>()); KryoSerializer<String> kryo = new KryoSerializer<>(String.class, conf); // we create Kryo from another type. assertTrue(kryo.getKryo().getRegistration(FromNested.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(ClassWithNested.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(Path.class).getId() > 0); // check if the generic type from one field is also registered (its very likely that // generic types are also used as fields somewhere. assertTrue(kryo.getKryo().getRegistration(FromGeneric1.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(FromGeneric2.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(Node.class).getId() > 0); } }
@Test public void testTypeRegistration() { ExecutionConfig conf = new ExecutionConfig(); Serializers.recursivelyRegisterType(ClassWithNested.class, conf, new HashSet<Class<?>>()); KryoSerializer<String> kryo = new KryoSerializer<>(String.class, conf); // we create Kryo from another type. Assert.assertTrue(kryo.getKryo().getRegistration(FromNested.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(ClassWithNested.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(Path.class).getId() > 0); // check if the generic type from one field is also registered (its very likely that // generic types are also used as fields somewhere. Assert.assertTrue(kryo.getKryo().getRegistration(FromGeneric1.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(FromGeneric2.class).getId() > 0); Assert.assertTrue(kryo.getKryo().getRegistration(Node.class).getId() > 0); // register again and make sure classes are still registered ExecutionConfig conf2 = new ExecutionConfig(); Serializers.recursivelyRegisterType(ClassWithNested.class, conf2, new HashSet<Class<?>>()); KryoSerializer<String> kryo2 = new KryoSerializer<>(String.class, conf); assertTrue(kryo2.getKryo().getRegistration(FromNested.class).getId() > 0); }
@Test public void testFoldingStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); @SuppressWarnings("unchecked") FoldFunction<String, TaskInfo> folder = (FoldFunction<String, TaskInfo>) mock(FoldFunction.class); FoldingStateDescriptor<String, TaskInfo> descr = new FoldingStateDescriptor<>("name", null, folder, TaskInfo.class); context.getFoldingState(descr); FoldingStateDescriptor<?, ?> descrIntercepted = (FoldingStateDescriptor<?, ?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void validateReferenceMappingEnabled() { KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); Kryo kryo = serializer.getKryo(); assertTrue(kryo.getReferences()); } }
private void validateStateDescriptorConfigured(SingleOutputStreamOperator<?> result) { OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation(); WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator(); StateDescriptor<?, ?> descr = op.getStateDescriptor(); // this would be the first statement to fail if state descriptors were not properly initialized TypeSerializer<?> serializer = descr.getSerializer(); assertTrue(serializer instanceof KryoSerializer); Kryo kryo = ((KryoSerializer<?>) serializer).getKryo(); assertTrue("serializer registration was not properly passed on", kryo.getSerializer(File.class) instanceof JavaSerializer); }
@Test public void testValueStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class); context.getState(descr); StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testReducingStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); @SuppressWarnings("unchecked") ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class); ReducingStateDescriptor<TaskInfo> descr = new ReducingStateDescriptor<>("name", reducer, TaskInfo.class); context.getReducingState(descr); StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testAggregatingStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); @SuppressWarnings("unchecked") AggregateFunction<String, TaskInfo, String> aggregate = (AggregateFunction<String, TaskInfo, String>) mock(AggregateFunction.class); AggregatingStateDescriptor<String, TaskInfo, String> descr = new AggregatingStateDescriptor<>("name", aggregate, TaskInfo.class); context.getAggregatingState(descr); AggregatingStateDescriptor<?, ?, ?> descrIntercepted = (AggregatingStateDescriptor<?, ?, ?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testMapStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); MapStateDescriptor<String, TaskInfo> descr = new MapStateDescriptor<>("name", String.class, TaskInfo.class); context.getMapState(descr); MapStateDescriptor<?, ?> descrIntercepted = (MapStateDescriptor<?, ?>) descriptorCapture.get(); TypeSerializer<?> valueSerializer = descrIntercepted.getValueSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(valueSerializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) valueSerializer).getKryo().getRegistration(Path.class).getId() > 0); }
int testClassId = kryoSerializer.getKryo().getRegistration(TestClass.class).getId(); int testClassAId = kryoSerializer.getKryo().getRegistration(TestClassA.class).getId(); int testClassBId = kryoSerializer.getKryo().getRegistration(TestClassB.class).getId(); kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializer); assertTrue(compatResult.isCompatibleAsIs()); assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId()); assertEquals(testClassAId, kryoSerializer.getKryo().getRegistration(TestClassA.class).getId()); assertEquals(testClassBId, kryoSerializer.getKryo().getRegistration(TestClassB.class).getId());
@Test public void testListStateInstantiation() throws Exception { final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(Path.class); final AtomicReference<Object> descriptorCapture = new AtomicReference<>(); StreamingRuntimeContext context = new StreamingRuntimeContext( createDescriptorCapturingMockOp(descriptorCapture, config), createMockEnvironment(), Collections.<String, Accumulator<?, ?>>emptyMap()); ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class); context.getListState(descr); ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get(); TypeSerializer<?> serializer = descrIntercepted.getSerializer(); // check that the Path class is really registered, i.e., the execution config was applied assertTrue(serializer instanceof ListSerializer); TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer(); assertTrue(elementSerializer instanceof KryoSerializer); assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0); }
@Test public void testInitializeSerializerAfterSerializationWithCustomConfig() throws Exception { // guard our test assumptions. assertEquals("broken test assumption", -1, new KryoSerializer<>(String.class, new ExecutionConfig()).getKryo() .getRegistration(File.class).getId()); final ExecutionConfig config = new ExecutionConfig(); config.registerKryoType(File.class); final TestStateDescriptor<Path> original = new TestStateDescriptor<>("test", Path.class); TestStateDescriptor<Path> clone = CommonTestUtils.createCopySerializable(original); clone.initializeSerializerUnlessSet(config); // serialized one (later initialized) carries the registration assertTrue(((KryoSerializer<?>) clone.getSerializer()).getKryo() .getRegistration(File.class).getId() > 0); }
private void validateListStateDescriptorConfigured(SingleOutputStreamOperator<?> result) { OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation(); WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator(); StateDescriptor<?, ?> descr = op.getStateDescriptor(); assertTrue(descr instanceof ListStateDescriptor); ListStateDescriptor<?> listDescr = (ListStateDescriptor<?>) descr; // this would be the first statement to fail if state descriptors were not properly initialized TypeSerializer<?> serializer = listDescr.getSerializer(); assertTrue(serializer instanceof ListSerializer); TypeSerializer<?> elementSerializer = listDescr.getElementSerializer(); assertTrue(elementSerializer instanceof KryoSerializer); Kryo kryo = ((KryoSerializer<?>) elementSerializer).getKryo(); assertTrue("serializer registration was not properly passed on", kryo.getSerializer(File.class) instanceof JavaSerializer); } }