return Types.FloatType.get(); case DOUBLE: return Types.DoubleType.get(); case STRING: case ENUM:
return Types.FloatType.get(); case DOUBLE: return Types.DoubleType.get(); case STRING: case CHAR:
return Types.FloatType.get(); case DOUBLE: return Types.DoubleType.get(); case FIXED_LEN_BYTE_ARRAY: return Types.FixedType.ofLength(primitive.getTypeLength());
), Types.StructType.of( required(12, "latitude", Types.DoubleType.get()), optional(25, "alt", Types.FloatType.get()) ))), ))), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get() )), optional(24, "toplevel", Types.DecimalType.of(9, 2)) .renameColumn("points.y", "y.y") // has a '.' in the field name .updateColumn("id", Types.LongType.get()) .updateColumn("locations.lat", Types.DoubleType.get()) // use the original name .deleteColumn("locations.long") .deleteColumn("properties")
Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(),
))), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get() )), optional(7, "properties", Types.MapType.ofOptional(18, 19,
))), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get() )), optional(7, "properties", Types.MapType.ofOptional(18, 19,
), Types.StructType.of( required(12, "lat", Types.DoubleType.get()), required(13, "long", Types.DoubleType.get()) ))), optional(5, "points", Types.ListType.ofOptional(14, ))), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get() )), optional(7, "properties", Types.MapType.ofOptional(18, 19, .updateColumn("locations.lat", Types.DoubleType.get()) .updateColumn("locations.long", Types.DoubleType.get()) .apply();
optional(22, "long", Types.LongType.get()), optional(23, "float", Types.FloatType.get()), optional(24, "double", Types.DoubleType.get()), optional(25, "date", Types.DateType.get()), optional(27, "time", Types.TimeType.get()),
))), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get() )), optional(7, "properties", Types.MapType.ofOptional(18, 19,
return Types.DoubleType.get();
14, Types.StructType.of(required(15, "x", Types.LongType.get()), required(16, "y", Types.LongType.get())))), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get())), optional(7, "properties", Types.MapType.ofOptional(18, 19, Types.StringType.get(), Types.StringType.get())));
Set<Pair<Type.PrimitiveType, Type.PrimitiveType>> allowedUpdates = Sets.newHashSet( Pair.of(Types.IntegerType.get(), Types.LongType.get()), Pair.of(Types.FloatType.get(), Types.DoubleType.get()), Pair.of(Types.DecimalType.of(9, 2), Types.DecimalType.of(18, 2)) ); Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.BinaryType.get(),
@Test public void mapConversions() throws IOException { // consistent behavior for maps conversions. The below test case, correctly does not specify map key types convertToPigSchema( new Schema( required( 1, "a", MapType.ofRequired( 2, 3, StringType.get(), ListType.ofRequired( 4, StructType.of( required(5, "b", LongType.get()), required(6, "c", StringType.get())))))), "a:[{(b:long,c:chararray)}]", "We do not specify the map key type here"); // struct<a:map<string,map<string,double>>> -> (a:[[double]]) // As per https://pig.apache.org/docs/latest/basic.html#map-schema. It seems that // we only need to specify value type as keys are always of type chararray convertToPigSchema( new Schema( StructType.of( required(1, "a", MapType.ofRequired( 2, 3, StringType.get(), MapType.ofRequired(4, 5, StringType.get(), DoubleType.get()))) ).fields()), "a:[[double]]", "A map key type does not need to be specified"); }
@Test public void testIdentityConversions() { List<Pair<Literal<?>, Type>> pairs = Arrays.asList( Pair.of(Literal.of(true), Types.BooleanType.get()), Pair.of(Literal.of(34), Types.IntegerType.get()), Pair.of(Literal.of(34L), Types.LongType.get()), Pair.of(Literal.of(34.11F), Types.FloatType.get()), Pair.of(Literal.of(34.55D), Types.DoubleType.get()), Pair.of(Literal.of("34.55"), Types.DecimalType.of(9, 2)), Pair.of(Literal.of("2017-08-18"), Types.DateType.get()), Pair.of(Literal.of("14:21:01.919"), Types.TimeType.get()), Pair.of(Literal.of("2017-08-18T14:21:01.919"), Types.TimestampType.withoutZone()), Pair.of(Literal.of("abc"), Types.StringType.get()), Pair.of(Literal.of(UUID.randomUUID()), Types.UUIDType.get()), Pair.of(Literal.of(new byte[] {0, 1, 2}), Types.FixedType.ofLength(3)), Pair.of(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BinaryType.get()) ); for (Pair<Literal<?>, Type> pair : pairs) { Literal<?> lit = pair.first(); Type type = pair.second(); // first, convert the literal to the target type (date/times start as strings) Literal<?> expected = lit.to(type); // then check that converting again to the same type results in an identical literal Assert.assertSame("Converting twice should produce identical values", expected, expected.to(type)); } }
public static boolean isPromotionAllowed(Type from, Type.PrimitiveType to) { // Warning! Before changing this function, make sure that the type change doesn't introduce // compatibility problems in partitioning. if (from.equals(to)) { return true; } switch (from.typeId()) { case INTEGER: return to == Types.LongType.get(); case FLOAT: return to == Types.DoubleType.get(); case DECIMAL: Types.DecimalType fromDecimal = (Types.DecimalType) from; if (to.typeId() != Type.TypeID.DECIMAL) { return false; } Types.DecimalType toDecimal = (Types.DecimalType) to; return (fromDecimal.scale() == toDecimal.scale() && fromDecimal.precision() <= toDecimal.precision()); } return false; }
@Test public void testIdentityTypes() throws Exception { // these types make a strong guarantee than equality, instances are identical Type[] identityPrimitives = new Type[] { Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withoutZone(), Types.TimestampType.withZone(), Types.StringType.get(), Types.UUIDType.get(), Types.BinaryType.get() }; for (Type type : identityPrimitives) { Assert.assertSame("Serialization result should be identical to starting type", type, TestHelpers.roundTripSerialize(type)); } }
@Test public void testInvalidTimeConversions() { testInvalidConversions( Literal.of("14:21:01.919").to(Types.TimeType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidDecimalConversions() { testInvalidConversions(Literal.of(new BigDecimal("34.11")), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidBinaryConversions() { testInvalidConversions(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1) ); }