@Override public Type map(Types.MapType map, Supplier<Type> keyFuture, Supplier<Type> valuefuture) { int newKeyId = nextId.get(); int newValueId = nextId.get(); if (map.isValueOptional()) { return Types.MapType.ofOptional(newKeyId, newValueId, keyFuture.get(), valuefuture.get()); } else { return Types.MapType.ofRequired(newKeyId, newValueId, keyFuture.get(), valuefuture.get()); } }
private static Types.MapType mapFromJson(JsonNode json) { int keyId = JsonUtil.getInt(KEY_ID, json); Type keyType = typeFromJson(json.get(KEY)); int valueId = JsonUtil.getInt(VALUE_ID, json); Type valueType = typeFromJson(json.get(VALUE)); boolean isRequired = JsonUtil.getBool(VALUE_REQUIRED, json); if (isRequired) { return Types.MapType.ofRequired(keyId, valueId, keyType, valueType); } else { return Types.MapType.ofOptional(keyId, valueId, keyType, valueType); } }
@Test public void testIncompatibleMapAndPrimitive() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); Schema read = new Schema(required(0, "map_field", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("map cannot be read as a string")); }
@Test public void testLongInBag() throws IOException { Schema icebergSchema = new Schema( optional( 1, "nested_list", MapType.ofOptional( 2, 3, StringType.get(), ListType.ofRequired(5, LongType.get())))); SchemaUtil.convert(icebergSchema); }
@Test public void nestedMaps() throws IOException { convertToPigSchema(new Schema( optional(1, "nested", MapType.ofOptional(2,3, StringType.get(), MapType.ofOptional(4,5, StringType.get(), MapType.ofOptional(6,7, StringType.get(), DecimalType.of(10,2))))) ),"nested:[[[bigdecimal]]]", ""); }
@Test public void testComplex() throws IOException { convertToPigSchema( new Schema( optional(1, "bag", ListType.ofOptional(2, BooleanType.get())), optional(3, "map", MapType.ofOptional(4,5, StringType.get(), DoubleType.get())), optional(6, "tuple", StructType.of(optional(7, "i", IntegerType.get()), optional(8,"f", FloatType.get()))) ),"bag:{(boolean)},map:[double],tuple:(i:int,f:float)", null ); }
@Override public Type map(MapType map, Type keyType, Type valueType) { if (map.valueContainsNull()) { return Types.MapType.ofOptional(getNextId(), getNextId(), keyType, valueType); } else { return Types.MapType.ofRequired(getNextId(), getNextId(), keyType, valueType); } }
@Override public Type map(Schema map, Type valueType) { Schema valueSchema = map.getValueType(); int keyId = getKeyId(map); int valueId = getValueId(map); if (AvroSchemaUtil.isOptionSchema(valueSchema)) { return Types.MapType.ofOptional(keyId, valueId, Types.StringType.get(), valueType); } else { return Types.MapType.ofRequired(keyId, valueId, Types.StringType.get(), valueType); } }
@Test public void testMap() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testNumericMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, LongType.get(), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testNumericMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.LongType.get(), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testUpdateMapKey() { Schema schema = new Schema(required(1, "m", Types.MapType.ofOptional(2, 3, Types.IntegerType.get(), Types.DoubleType.get()))); AssertHelpers.assertThrows("Should reject update map key", IllegalArgumentException.class, "Cannot update map keys", () -> { new SchemaUpdate(schema, 3).updateColumn("m.key", Types.LongType.get()).apply(); } ); } }
@Test public void testNumericMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.LongType.get(), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test(expected = FrontendException.class) public void invalidMap() throws IOException { convertToPigSchema(new Schema( optional(1, "invalid", MapType.ofOptional(2,3, IntegerType.get(), DoubleType.get())) ), "", ""); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testMap() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), Types.StringType.get()))); writeAndValidate(schema); }