@Test public void testTruncateByteBuffer() throws Exception { Truncate<ByteBuffer> trunc = Truncate.get(Types.BinaryType.get(), 4); Assert.assertEquals("Should truncate binary longer than length", ByteBuffer.wrap("abcd".getBytes("UTF-8")), trunc.apply(ByteBuffer.wrap("abcdefg".getBytes("UTF-8")))); Assert.assertEquals("Should not pad binary shorter than length", ByteBuffer.wrap("abc".getBytes("UTF-8")), trunc.apply(ByteBuffer.wrap("abc".getBytes("UTF-8")))); } }
@Test public void testByteBufferOffHeap() { byte[] bytes = randomBytes(128); ByteBuffer buffer = ByteBuffer.allocateDirect(128); // copy to the middle of the off-heap buffer buffer.position(5); buffer.limit(105); buffer.mark(); buffer.put(bytes, 5, 100); buffer.reset(); Bucket<ByteBuffer> bucketFunc = Bucket.get(Types.BinaryType.get(), 100); Assert.assertEquals( "DirectByteBuffer hash should match hash for correct slice", hashBytes(bytes, 5, 100), bucketFunc.hash(buffer)); // verify that the buffer was not modified Assert.assertEquals("Buffer position should not change", 5, buffer.position()); Assert.assertEquals("Buffer limit should not change", 105, buffer.limit()); }
@Test public void testByteBufferOnHeap() { byte[] bytes = randomBytes(128); ByteBuffer buffer = ByteBuffer.wrap(bytes, 5, 100); Bucket<ByteBuffer> bucketFunc = Bucket.get(Types.BinaryType.get(), 100); Assert.assertEquals( "HeapByteBuffer hash should match hash for correct slice", hashBytes(bytes, 5, 100), bucketFunc.hash(buffer)); // verify that the buffer was not modified Assert.assertEquals("Buffer position should not change", 5, buffer.position()); Assert.assertEquals("Buffer limit should not change", 105, buffer.limit()); }
@Test public void testInvalidDecimalConversions() { testInvalidConversions(Literal.of(new BigDecimal("34.11")), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidTimeConversions() { testInvalidConversions( Literal.of("14:21:01.919").to(Types.TimeType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidDateConversions() { testInvalidConversions(Literal.of("2017-08-18").to(Types.DateType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidBooleanConversions() { testInvalidConversions(Literal.of(true), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void complexNested() throws IOException { convertToPigSchema(new Schema( optional(1,"t", StructType.of( optional(2, "b", ListType.ofOptional(3,StructType.of( optional(4, "i", IntegerType.get()), optional(5,"s", StringType.get()) ))) )), optional(6, "m1", MapType.ofOptional(7,8, StringType.get(), StructType.of( optional(9, "b", ListType.ofOptional(10, BinaryType.get()) ), optional(11, "m2", MapType.ofOptional(12,13, StringType.get(), IntegerType.get())) ))), optional(14, "b1", ListType.ofOptional(15, MapType.ofOptional(16,17, StringType.get(), ListType.ofOptional(18, FloatType.get())))) ), "t:(b:{(i:int,s:chararray)}),m1:[(b:{(bytearray)},m2:[int])],b1:{([{(float)}])}", ""); }
@Test public void testInvalidUUIDConversions() { testInvalidConversions(Literal.of(UUID.randomUUID()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidDoubleConversions() { testInvalidConversions(Literal.of(34.11D), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidFloatConversions() { testInvalidConversions(Literal.of(34.11F), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidTimestampConversions() { testInvalidConversions( Literal.of("2017-08-18T14:21:01.919").to(Types.TimestampType.withoutZone()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.TimeType.get(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testPrimitive() throws IOException { Schema icebergSchema = new Schema( optional(1, "b", BooleanType.get()), optional(1, "i", IntegerType.get()), optional(2, "l", LongType.get()), optional(3, "f", FloatType.get()), optional(4, "d", DoubleType.get()), optional(5, "dec", DecimalType.of(0,2)), optional(5, "s", StringType.get()), optional(6,"bi", BinaryType.get()) ); ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema); assertEquals("b:boolean,i:int,l:long,f:float,d:double,dec:bigdecimal,s:chararray,bi:bytearray", pigSchema.toString()); }
@Test public void testInvalidStringConversions() { // Strings can be used for types that are difficult to construct, like decimal or timestamp, // but are not intended to support parsing strings to any type testInvalidConversions(Literal.of("abc"), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidIntegerConversions() { testInvalidConversions(Literal.of(34), Types.BooleanType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testMapOfLongToBytes() { Type map = Types.MapType.ofRequired(33, 34, Types.LongType.get(), Types.BinaryType.get()); Schema schema = AvroSchemaUtil.createMap( 33, Schema.create(Schema.Type.LONG), 34, Schema.create(Schema.Type.BYTES)); Assert.assertEquals("Avro schema to map", map, AvroSchemaUtil.convert(schema)); Assert.assertEquals("Map to Avro schema", schema, AvroSchemaUtil.convert(map)); }
@Test public void testMapOfStringToBytes() { Type map = Types.MapType.ofRequired(33, 34, Types.StringType.get(), Types.BinaryType.get()); Schema schema = addKeyId(33, addValueId(34, SchemaBuilder.map().values( Schema.create(Schema.Type.BYTES)))); Assert.assertEquals("Avro schema to map", map, AvroSchemaUtil.convert(schema)); Assert.assertEquals("Map to Avro schema", schema, AvroSchemaUtil.convert(map)); }
@Test public void testFixedToBinary() { Literal<ByteBuffer> lit = Literal.of(new byte[] {0, 1, 2}); Literal<ByteBuffer> binaryLit = lit.to(Types.BinaryType.get()); Assert.assertNotNull("Should allow conversion to binary", binaryLit); Assert.assertEquals("Conversion should not change value", lit.value().duplicate(), binaryLit.value().duplicate()); }
@Test public void testBinaryHumanString() { Types.BinaryType binary = Types.BinaryType.get(); Transform<ByteBuffer, ByteBuffer> identity = Transforms.identity(binary); Assert.assertEquals("Should base64-encode binary", "AQID", identity.toHumanString(ByteBuffer.wrap(new byte[] {1, 2, 3}))); }
@Test public void testInvalidLongConversions() { testInvalidConversions(Literal.of(34L), Types.BooleanType.get(), Types.DateType.get(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }