@Test public void testIncompatibleListElement() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.StringType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to string")); }
@Test @SuppressWarnings("unchecked") public void testIsNull() { StructType optional = StructType.of(optional(19, "s", Types.StringType.get())); UnboundPredicate<?> unbound = new UnboundPredicate<>(IS_NULL, ref("s")); Expression expr = unbound.bind(optional); BoundPredicate<?> bound = assertAndUnwrap(expr); Assert.assertEquals("Should use the same operation", IS_NULL, bound.op()); Assert.assertEquals("Should use the correct field", 19, bound.ref().fieldId()); Assert.assertNull("Should not have a literal value", bound.literal()); StructType required = StructType.of(required(20, "s", Types.StringType.get())); Assert.assertEquals("IsNull inclusive a required field should be alwaysFalse", Expressions.alwaysFalse(), unbound.bind(required)); }
@Test public void testStructs() throws Exception { Types.StructType struct = Types.StructType.of( Types.NestedField.required(34, "Name!", Types.StringType.get()), Types.NestedField.optional(35, "col", Types.DecimalType.of(38, 2))); Type copy = TestHelpers.roundTripSerialize(struct); Assert.assertEquals("Struct serialization should be equal to starting type", struct, copy); Type stringType = copy.asNestedType().asStructType().fieldType("Name!"); Assert.assertSame("Struct serialization should preserve identity type", Types.StringType.get(), stringType); Type decimalType = copy.asNestedType().asStructType().field(35).type(); Assert.assertEquals("Struct serialization should support id lookup", Types.DecimalType.of(38, 2), decimalType); }
@Test public void testMaps() throws Exception { Type[] maps = new Type[] { Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.LongType.get()), Types.MapType.ofRequired(4, 5, Types.StringType.get(), Types.LongType.get()) }; for (Type map : maps) { Type copy = TestHelpers.roundTripSerialize(map); Assert.assertEquals("Map serialization should be equal to starting type", map, copy); Assert.assertSame("Map serialization should preserve identity type", Types.LongType.get(), map.asNestedType().asMapType().valueType()); } }
@Test public void testInvalidIntegerConversions() { testInvalidConversions(Literal.of(34), Types.BooleanType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testIncompatibleMapValue() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); Schema read = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.DoubleType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to double")); }
@Test public void testIncompatibleMapKey() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.IntegerType.get(), Types.StringType.get() ))); Schema read = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.DoubleType.get(), Types.StringType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to double")); }
@Test public void testRequiredMapValue() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); Schema read = new Schema(required(0, "map_field", Types.MapType.ofRequired( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that values are optional", errors.get(0).contains("values should be required, but are optional")); }
@Test public void nestedTuples() throws IOException { convertToPigSchema(new Schema( optional(1,"first", StructType.of( optional(2, "second", StructType.of( optional(3, "third", StructType.of( optional(4, "val", StringType.get()) )) )) )) ), "first:(second:(third:(val:chararray)))", ""); }
@Test public void testStringToStringLiteral() { Literal<CharSequence> string = Literal.of("abc"); Assert.assertSame("Should return same instance", string, string.to(Types.StringType.get())); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testMapOfStringToBytes() { Type map = Types.MapType.ofRequired(33, 34, Types.StringType.get(), Types.BinaryType.get()); Schema schema = addKeyId(33, addValueId(34, SchemaBuilder.map().values( Schema.create(Schema.Type.BYTES)))); Assert.assertEquals("Avro schema to map", map, AvroSchemaUtil.convert(schema)); Assert.assertEquals("Map to Avro schema", schema, AvroSchemaUtil.convert(map)); }
@Test public void testAddFieldToMapKey() { AssertHelpers.assertThrows("Should reject add sub-field to map key", IllegalArgumentException.class, "Cannot add fields to map keys", () -> { new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID) .addColumn("locations.key", "address_line_2", Types.StringType.get()).apply(); } ); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testUtf8() { Utf8 utf8 = new Utf8("string to test murmur3 hash"); byte[] asBytes = utf8.toString().getBytes(Charsets.UTF_8); Bucket<CharSequence> bucketFunc = Bucket.get(Types.StringType.get(), 100); Assert.assertEquals("String hash should match hash of UTF-8 bytes", hashBytes(asBytes), bucketFunc.hash(utf8)); }
@Test public void testIncompatibleListAndPrimitive() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("list cannot be read as a string")); }
@Test public void testArray() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }