@Test public void testInteger() { int num = testRandom.nextInt(); ByteBuffer buffer = ByteBuffer.allocate(8); buffer.order(ByteOrder.LITTLE_ENDIAN); buffer.putLong((long) num); Bucket<Integer> bucketFunc = Bucket.get(Types.IntegerType.get(), 100); Assert.assertEquals("Integer hash should match hash of little-endian bytes", hashBytes(buffer.array()), bucketFunc.hash(num)); }
@Test public void testRequiredSchemaField() { Schema write = new Schema(optional(1, "from_field", Types.IntegerType.get())); Schema read = new Schema(required(1, "to_field", Types.IntegerType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that a required column is optional", errors.get(0).contains("should be required, but is optional")); }
@Test public void testMissingSchemaField() { Schema write = new Schema(required(0, "other_field", Types.IntegerType.get())); Schema read = new Schema(required(1, "to_field", Types.IntegerType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that a required column is missing", errors.get(0).contains("is required, but is missing")); }
@Test public void testIncompatibleMapKey() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.IntegerType.get(), Types.StringType.get() ))); Schema read = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.DoubleType.get(), Types.StringType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to double")); }
@Test public void testRequiredMapValue() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); Schema read = new Schema(required(0, "map_field", Types.MapType.ofRequired( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that values are optional", errors.get(0).contains("values should be required, but are optional")); }
@Test public void testIncompatibleMapValue() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); Schema read = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.DoubleType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to double")); }
@Test public void testMissingRequiredStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( optional(2, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.IntegerType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that a required field is missing", errors.get(0).contains("is required, but is missing")); }
@Test public void testIncompatibleMapAndPrimitive() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); Schema read = new Schema(required(0, "map_field", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("map cannot be read as a string")); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testIncompatibleListElement() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.StringType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to string")); }
@Test(expected = FrontendException.class) public void invalidMap() throws IOException { convertToPigSchema(new Schema( optional(1, "invalid", MapType.ofOptional(2,3, IntegerType.get(), DoubleType.get())) ), "", ""); }
@Test public void testIncompatibleListAndPrimitive() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("list cannot be read as a string")); }
@Test public void testComplex() throws IOException { convertToPigSchema( new Schema( optional(1, "bag", ListType.ofOptional(2, BooleanType.get())), optional(3, "map", MapType.ofOptional(4,5, StringType.get(), DoubleType.get())), optional(6, "tuple", StructType.of(optional(7, "i", IntegerType.get()), optional(8,"f", FloatType.get()))) ),"bag:{(boolean)},map:[double],tuple:(i:int,f:float)", null ); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testLongToIntegerConversion() { Literal<Long> lit = Literal.of(34L); Literal<Integer> intLit = lit.to(Types.IntegerType.get()); Assert.assertEquals("Value should match", 34, (int) intLit.value()); Assert.assertEquals("Values above Integer.MAX_VALUE should be Literals.aboveMax()", Literals.aboveMax(), Literal.of((long) Integer.MAX_VALUE + 1L).to(Types.IntegerType.get())); Assert.assertEquals("Values below Integer.MIN_VALUE should be Literals.belowMin()", Literals.belowMin(), Literal.of((long) Integer.MIN_VALUE - 1L).to(Types.IntegerType.get())); }
private static Schema wrapFileSchema(StructType fileStruct) { // ids for top-level columns are assigned from 1000 return new Schema( required(0, "status", IntegerType.get()), required(1, "snapshot_id", LongType.get()), required(2, "data_file", fileStruct)); }
@Test public void testMissingOptionalStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( required(2, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( optional(1, "to_field", Types.IntegerType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce no error messages", 0, errors.size()); }
@Test public void testUpdateMapKey() { Schema schema = new Schema(required(1, "m", Types.MapType.ofOptional(2, 3, Types.IntegerType.get(), Types.DoubleType.get()))); AssertHelpers.assertThrows("Should reject update map key", IllegalArgumentException.class, "Cannot update map keys", () -> { new SchemaUpdate(schema, 3).updateColumn("m.key", Types.LongType.get()).apply(); } ); } }
@Test public void testIntegerTypePromotion() { Bucket<Integer> bucketInts = Bucket.get(Types.IntegerType.get(), 100); Bucket<Long> bucketLongs = Bucket.get(Types.LongType.get(), 100); int r = testRandom.nextInt(); Assert.assertEquals("Integer and Long bucket results should match", bucketInts.apply(r), bucketLongs.apply((long) r)); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }