/** * Returns a type descriptor for the input of the given {@link SerializableFunction}, subject to * Java type erasure: may contain unresolved type variables if the type was erased. */ public static <InputT, OutputT> TypeDescriptor<InputT> inputOf( SerializableFunction<InputT, OutputT> fn) { return extractFromTypeParameters( fn, SerializableFunction.class, new TypeVariableExtractor<SerializableFunction<InputT, OutputT>, InputT>() {}); }
/** Get a {@link TypeDescriptor} from a {@link FieldType}. */ public static TypeDescriptor javaTypeForFieldType(FieldType fieldType) { switch (fieldType.getTypeName()) { case ARRAY: return TypeDescriptors.lists(javaTypeForFieldType(fieldType.getCollectionElementType())); case MAP: return TypeDescriptors.maps( javaTypeForFieldType(fieldType.getMapKeyType()), javaTypeForFieldType(fieldType.getMapValueType())); case ROW: return TypeDescriptors.rows(); default: return PRIMITIVE_MAPPING.get(fieldType.getTypeName()); } } /** Get a {@link FieldType} from a {@link TypeDescriptor}. */
@Test public void testPrimitiveTypeToJavaType() { assertEquals( TypeDescriptors.bytes(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.BYTE)); assertEquals( TypeDescriptors.shorts(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.INT16)); assertEquals( TypeDescriptors.integers(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.INT32)); assertEquals( TypeDescriptors.longs(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.INT64)); assertEquals( TypeDescriptors.bigdecimals(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.DECIMAL)); assertEquals( TypeDescriptors.floats(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.FLOAT)); assertEquals( TypeDescriptors.doubles(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.DOUBLE)); assertEquals( TypeDescriptors.strings(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.STRING)); assertEquals( TypeDescriptor.of(Instant.class), FieldTypeDescriptors.javaTypeForFieldType(FieldType.DATETIME)); assertEquals( TypeDescriptors.booleans(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.BOOLEAN)); assertEquals( TypeDescriptor.of(byte[].class), FieldTypeDescriptors.javaTypeForFieldType(FieldType.BYTES)); }
@Override public PCollection<String> expand(PCollection<Row> input) { return input.apply( "rowToCsv", MapElements.into(TypeDescriptors.strings()).via(row -> beamRow2CsvLine(row, csvFormat))); } }
@Test public void testMapTypeToJavaType() { assertEquals( TypeDescriptors.maps(TypeDescriptors.strings(), TypeDescriptors.longs()), FieldTypeDescriptors.javaTypeForFieldType( FieldType.map(FieldType.STRING, FieldType.INT64))); assertEquals( TypeDescriptors.maps( TypeDescriptors.strings(), TypeDescriptors.lists(TypeDescriptors.longs())), FieldTypeDescriptors.javaTypeForFieldType( FieldType.map(FieldType.STRING, FieldType.array(FieldType.INT64)))); }
@Test public void testTypeDescriptorsListsOfLists() throws Exception { TypeDescriptor<List<List<String>>> descriptor = lists(lists(strings())); assertEquals(descriptor, new TypeDescriptor<List<List<String>>>() {}); assertNotEquals(descriptor, new TypeDescriptor<List<String>>() {}); assertNotEquals(descriptor, new TypeDescriptor<List<Boolean>>() {}); }
checkArgument( inputTypeDescriptor.equals( TypeDescriptors.kvs(outputFormatKeyClass, outputFormatValueClass)), "%s expects following %ss: KV(Key: %s, Value: %s) but following %ss are set: KV(Key: %s, Value: %s)", Write.class.getSimpleName(),
@Override public PCollection<KV<Integer, KV<KeyT, ValueT>>> expand(PCollection<KV<KeyT, ValueT>> input) { return input .apply( "AssignTask", ParDo.of(new AssignTaskFn<KeyT, ValueT>(configView)).withSideInputs(configView)) .setTypeDescriptor( TypeDescriptors.kvs(TypeDescriptors.integers(), input.getTypeDescriptor())) .apply("GroupByTaskId", GroupByKey.create()) .apply("FlattenGroupedTasks", ParDo.of(new FlattenGroupedTasks<>())); } }
@Test public void testCoderProvidersFromStaticMethodsForParameterlessTypes() throws Exception { CoderProvider factory = CoderProviders.fromStaticMethods(String.class, StringUtf8Coder.class); assertEquals( StringUtf8Coder.of(), factory.coderFor(TypeDescriptors.strings(), Collections.emptyList())); factory = CoderProviders.fromStaticMethods(Double.class, DoubleCoder.class); assertEquals( DoubleCoder.of(), factory.coderFor(TypeDescriptors.doubles(), Collections.emptyList())); factory = CoderProviders.fromStaticMethods(byte[].class, ByteArrayCoder.class); assertEquals( ByteArrayCoder.of(), factory.coderFor(TypeDescriptor.of(byte[].class), Collections.emptyList())); }
TypeDescriptors.iterables(TypeDescriptors.integers()); ParDo.of(new WriteFn<KeyT, ValueT>(configView, externalSynchronization)) .withSideInputs(configView)) .setTypeDescriptor(TypeDescriptors.integers()) .apply( "CollectWriteTasks", Combine.globally(new IterableCombinerFn<>(TypeDescriptors.integers())) .withoutDefaults()) .setTypeDescriptor(iterableIntType);
/** * Checks that {#link CoderProviders.fromStaticMethods} successfully builds a working {@link * CoderProvider} from {@link KvCoder KvCoder.class}. */ @Test public void testKvCoderProvider() throws Exception { TypeDescriptor<KV<Double, Double>> type = TypeDescriptors.kvs(TypeDescriptors.doubles(), TypeDescriptors.doubles()); CoderProvider kvCoderProvider = CoderProviders.fromStaticMethods(KV.class, KvCoder.class); assertEquals( KvCoder.of(DoubleCoder.of(), DoubleCoder.of()), kvCoderProvider.coderFor(type, Arrays.asList(DoubleCoder.of(), DoubleCoder.of()))); }
@Override public PCollection<Row> expand(PCollection<String> input) { return input .apply( "csvToRow", FlatMapElements.into(TypeDescriptors.rows()) .via(s -> csvLines2BeamRows(csvFormat, s, schema))) .setRowSchema(schema); } }