/** * Returns generic type information for any Java object. The serialization logic will * use the general purpose serializer Kryo. * * <p>Generic types are black-boxes for Flink, but allow any object and null values in fields. * * <p>By default, serialization of this type is not very efficient. Please read the documentation * about how to improve efficiency (namely by pre-registering classes). * * @param genericClass any Java class */ public static <T> TypeInformation<T> GENERIC(Class<T> genericClass) { return new GenericTypeInfo<>(genericClass); }
@Override protected GenericTypeInfo<?>[] getTestData() { return new GenericTypeInfo<?>[] { new GenericTypeInfo<>(TestClass.class), new GenericTypeInfo<>(AlternativeClass.class) }; }
private TypeInformation getFieldType(HCatFieldSchema fieldSchema) { switch(fieldSchema.getType()) { case INT: return BasicTypeInfo.INT_TYPE_INFO; case TINYINT: return BasicTypeInfo.BYTE_TYPE_INFO; case SMALLINT: return BasicTypeInfo.SHORT_TYPE_INFO; case BIGINT: return BasicTypeInfo.LONG_TYPE_INFO; case BOOLEAN: return BasicTypeInfo.BOOLEAN_TYPE_INFO; case FLOAT: return BasicTypeInfo.FLOAT_TYPE_INFO; case DOUBLE: return BasicTypeInfo.DOUBLE_TYPE_INFO; case STRING: return BasicTypeInfo.STRING_TYPE_INFO; case BINARY: return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO; case ARRAY: return new GenericTypeInfo(List.class); case MAP: return new GenericTypeInfo(Map.class); case STRUCT: return new GenericTypeInfo(List.class); default: throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered."); } }
"and must be processed as GenericType. Please read the Flink documentation " + "on \"Data Types & Serialization\" for details of the effect on performance."); return new GenericTypeInfo<OUT>(clazz); "and must be processed as GenericType. Please read the Flink documentation " + "on \"Data Types & Serialization\" for details of the effect on performance."); return new GenericTypeInfo<OUT>(clazz); genericClass = typeToClass(fieldType); pojoFields.add(new PojoField(field, new GenericTypeInfo<OUT>((Class<OUT>) genericClass)));
@SuppressWarnings("unchecked") @Internal private static <T extends SpecificRecordBase> List<PojoField> generateFieldsFromAvroSchema(Class<T> typeClass) { PojoTypeExtractor pte = new PojoTypeExtractor(); ArrayList<Type> typeHierarchy = new ArrayList<>(); typeHierarchy.add(typeClass); TypeInformation ti = pte.analyzePojo(typeClass, typeHierarchy, null, null, null); if (!(ti instanceof PojoTypeInfo)) { throw new IllegalStateException("Expecting type to be a PojoTypeInfo"); } PojoTypeInfo pti = (PojoTypeInfo) ti; List<PojoField> newFields = new ArrayList<>(pti.getTotalFields()); for (int i = 0; i < pti.getArity(); i++) { PojoField f = pti.getPojoFieldAt(i); TypeInformation newType = f.getTypeInformation(); // check if type is a CharSequence if (newType instanceof GenericTypeInfo) { if ((newType).getTypeClass().equals(CharSequence.class)) { // replace the type by a org.apache.avro.util.Utf8 newType = new GenericTypeInfo(org.apache.avro.util.Utf8.class); } } PojoField newField = new PojoField(f.getField(), newType); newFields.add(newField); } return newFields; }
@Test(expected = InvalidProgramException.class) public void testGenericNonKeyType() { // Fail: GenericType cannot be used as key TypeInformation<GenericNonKeyType> genericType = new GenericTypeInfo<>(GenericNonKeyType.class); new ExpressionKeys<>("*", genericType); }
@Override protected ObjectArrayTypeInfo<?, ?>[] getTestData() { return new ObjectArrayTypeInfo<?, ?>[] { ObjectArrayTypeInfo.getInfoFor(TestClass[].class, new GenericTypeInfo<>(TestClass.class)), ObjectArrayTypeInfo.getInfoFor(TestClass[].class, new PojoTypeInfo<>(TestClass.class, new ArrayList<PojoField>())) }; }
@Test public void testForwardWithGenericTypePublicAttrAccess() { compareAnalyzerResultWithAnnotationsSingleInput(MapFunction.class, Map4.class, new GenericTypeInfo<>(MyPojo.class), Types.STRING); }
return new GenericTypeInfo<>(clazz); return new GenericTypeInfo<OUT>(clazz); return new GenericTypeInfo<>(clazz); return new GenericTypeInfo<OUT>(clazz); return new GenericTypeInfo<OUT>(clazz);
@Override protected <T> TypeSerializer<T> createSerializer(Class<T> type) { ExecutionConfig conf = new ExecutionConfig(); conf.registerTypeWithKryoSerializer(LocalDate.class, LocalDateSerializer.class); TypeInformation<T> typeInfo = new GenericTypeInfo<T>(type); return typeInfo.createSerializer(conf); }
@Test public void testOfGenericClassForGenericType() { assertEquals(new GenericTypeInfo<>(List.class), TypeInformation.of(List.class)); }
new GenericTypeInfo<>(FileCopyTask.class), "fileCopyTasks");
@Test public void testObjectArrayKeyRejection() { KeySelector<Tuple2<Integer[], String>, Object[]> keySelector = new KeySelector<Tuple2<Integer[], String>, Object[]>() { @Override public Object[] getKey(Tuple2<Integer[], String> value) throws Exception { Object[] ks = new Object[value.f0.length]; for (int i = 0; i < ks.length; i++) { ks[i] = new Object(); } return ks; } }; ObjectArrayTypeInfo<Object[], Object> keyTypeInfo = ObjectArrayTypeInfo.getInfoFor( Object[].class, new GenericTypeInfo<>(Object.class)); testKeyRejection(keySelector, keyTypeInfo); }
@Test public void testKeyGenericType() { TypeInformation<GenericKeyType> genericType = new GenericTypeInfo<>(GenericKeyType.class); ExpressionKeys<GenericKeyType> ek = new ExpressionKeys<>("*", genericType); Assert.assertArrayEquals(new int[] {0}, ek.computeLogicalKeyPositions()); }
/** * Test if the TypeExtractor is accepting untyped generics, * making them GenericTypes */ @Test public void testPojoWithGenericsSomeFieldsGeneric() { TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(PojoWithGenerics.class); Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>); PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass; for(int i = 0; i < pojoTypeForClass.getArity(); i++) { PojoField field = pojoTypeForClass.getPojoFieldAt(i); String name = field.getField().getName(); if(name.equals("field1")) { Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.getTypeInformation()); } else if (name.equals("field2")) { Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.getTypeInformation()); } else if (name.equals("key")) { Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.getTypeInformation()); } else { Assert.fail("Unexpected field "+field); } } }
@Test public void testDisableGenericTypes() { ExecutionConfig conf = new ExecutionConfig(); TypeInformation<Object> typeInfo = new GenericTypeInfo<Object>(Object.class); // by default, generic types are supported TypeSerializer<Object> serializer = typeInfo.createSerializer(conf); assertTrue(serializer instanceof KryoSerializer); // expect an exception when generic types are disabled conf.disableGenericTypes(); try { typeInfo.createSerializer(conf); fail("should have failed with an exception"); } catch (UnsupportedOperationException e) { // expected } }
@Test public void testRow() { Row row = new Row(2); row.setField(0, "string"); row.setField(1, 15); TypeInformation<Row> rowInfo = TypeExtractor.getForObject(row); Assert.assertEquals(rowInfo.getClass(), RowTypeInfo.class); Assert.assertEquals(2, rowInfo.getArity()); Assert.assertEquals( new RowTypeInfo( BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), rowInfo); Row nullRow = new Row(2); TypeInformation<Row> genericRowInfo = TypeExtractor.getForObject(nullRow); Assert.assertEquals(genericRowInfo, new GenericTypeInfo<>(Row.class)); }
@Test public void testJoinWithAtomicType1() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env); DataSet<Integer> ds2 = env.fromElements(1, 2); DataSet<Tuple2<Tuple3<Integer, Long, String>, Integer>> joinDs = ds1 .fullOuterJoin(ds2) .where(0) .equalTo("*") .with(new ProjectBothFunction<Tuple3<Integer, Long, String>, Integer>()) .returns(new GenericTypeInfo(Tuple2.class)); List<Tuple2<Tuple3<Integer, Long, String>, Integer>> result = joinDs.collect(); String expected = "(1,1,Hi),1\n" + "(2,2,Hello),2\n" + "(3,2,Hello world),null\n"; compareResultAsTuples(result, expected); }
@Test public void testJoinWithAtomicType2() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Integer> ds1 = env.fromElements(1, 2); DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env); DataSet<Tuple2<Integer, Tuple3<Integer, Long, String>>> joinDs = ds1 .fullOuterJoin(ds2) .where("*") .equalTo(0) .with(new ProjectBothFunction<Integer, Tuple3<Integer, Long, String>>()) .returns(new GenericTypeInfo(Tuple2.class)); List<Tuple2<Integer, Tuple3<Integer, Long, String>>> result = joinDs.collect(); String expected = "1,(1,1,Hi)\n" + "2,(2,2,Hello)\n" + "null,(3,2,Hello world)\n"; compareResultAsTuples(result, expected); }
@Test public void testTypeRegistrationFromTypeInfo() { ExecutionConfig conf = new ExecutionConfig(); Serializers.recursivelyRegisterType(new GenericTypeInfo<>(ClassWithNested.class), conf, new HashSet<Class<?>>()); KryoSerializer<String> kryo = new KryoSerializer<>(String.class, conf); // we create Kryo from another type. assertTrue(kryo.getKryo().getRegistration(FromNested.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(ClassWithNested.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(Path.class).getId() > 0); // check if the generic type from one field is also registered (its very likely that // generic types are also used as fields somewhere. assertTrue(kryo.getKryo().getRegistration(FromGeneric1.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(FromGeneric2.class).getId() > 0); assertTrue(kryo.getKryo().getRegistration(Node.class).getId() > 0); } }