@Override public Type union(Schema union, List<Type> options) { Preconditions.checkArgument(AvroSchemaUtil.isOptionSchema(union), "Unsupported type: non-option union: {}", union); // records, arrays, and maps will check nullability later if (options.get(0) == null) { return options.get(1); } else { return options.get(0); } }
static Schema toOption(Schema schema) { if (schema.getType() == UNION) { Preconditions.checkArgument(isOptionSchema(schema), "Union schemas are not supported: " + schema); return schema; } else { return Schema.createUnion(NULL, schema); } }
@Override public Schema union(Schema union, List<Schema> options) { Preconditions.checkState(AvroSchemaUtil.isOptionSchema(union), "Invalid schema: non-option unions are not supported: {}", union); // only unions with null are allowed, and a null schema results in null Schema pruned = null; if (options.get(0) != null) { pruned = options.get(0); } else if (options.get(1) != null) { pruned = options.get(1); } if (pruned != null) { if (pruned != AvroSchemaUtil.fromOption(union)) { return AvroSchemaUtil.toOption(pruned); } return union; } return null; }
@Override public Schema union(Schema union, Iterable<Schema> options) { Preconditions.checkState(isOptionSchema(union), "Invalid schema: non-option unions are not supported: {}", union); Schema nonNullOriginal = fromOption(union); Schema nonNullResult = fromOptions(Lists.newArrayList(options)); if (nonNullOriginal != nonNullResult) { return toOption(nonNullResult); } return union; }
@Override public Type record(Schema record, List<String> names, List<Type> fieldTypes) { List<Schema.Field> fields = record.getFields(); List<Types.NestedField> newFields = Lists.newArrayListWithExpectedSize(fields.size()); if (root == record) { this.nextId = 0; } for (int i = 0; i < fields.size(); i += 1) { Schema.Field field = fields.get(i); Type fieldType = fieldTypes.get(i); int fieldId = getId(field); if (AvroSchemaUtil.isOptionSchema(field.schema())) { newFields.add(Types.NestedField.optional(fieldId, field.name(), fieldType)); } else { newFields.add(Types.NestedField.required(fieldId, field.name(), fieldType)); } } return Types.StructType.of(newFields); }
static Schema createMap(int keyId, Schema keySchema, int valueId, Schema valueSchema) { String keyValueName = "k" + keyId + "_v" + valueId; Schema.Field keyField = new Schema.Field("key", keySchema, null, null); keyField.addProp(FIELD_ID_PROP, keyId); Schema.Field valueField = new Schema.Field("value", valueSchema, null, isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE: null); valueField.addProp(FIELD_ID_PROP, valueId); return LogicalMap.get().addToSchema(Schema.createArray(Schema.createRecord( keyValueName, null, null, false, ImmutableList.of(keyField, valueField)))); }
static Schema createProjectionMap(String recordName, int keyId, String keyName, Schema keySchema, int valueId, String valueName, Schema valueSchema) { String keyValueName = "k" + keyId + "_v" + valueId; Schema.Field keyField = new Schema.Field("key", keySchema, null, null); if (!"key".equals(keyName)) { keyField.addAlias(keyName); } keyField.addProp(FIELD_ID_PROP, keyId); Schema.Field valueField = new Schema.Field("value", valueSchema, null, isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE: null); valueField.addProp(FIELD_ID_PROP, valueId); if (!"value".equals(valueName)) { valueField.addAlias(valueName); } Schema keyValueRecord = Schema.createRecord( keyValueName, null, null, false, ImmutableList.of(keyField, valueField)); if (!keyValueName.equals(recordName)) { keyValueRecord.addAlias(recordName); } return LogicalMap.get().addToSchema(Schema.createArray(keyValueRecord)); }
@Override public Type map(Schema map, Type valueType) { Schema valueSchema = map.getValueType(); int keyId = getKeyId(map); int valueId = getValueId(map); if (AvroSchemaUtil.isOptionSchema(valueSchema)) { return Types.MapType.ofOptional(keyId, valueId, Types.StringType.get(), valueType); } else { return Types.MapType.ofRequired(keyId, valueId, Types.StringType.get(), valueType); } }
@Override public Type array(Schema array, Type elementType) { if (array.getLogicalType() instanceof LogicalMap) { // map stored as an array Schema keyValueSchema = array.getElementType(); Preconditions.checkArgument(AvroSchemaUtil.isKeyValueSchema(keyValueSchema), "Invalid key-value pair schema: {}", keyValueSchema); Types.StructType keyValueType = elementType.asStructType(); Types.NestedField keyField = keyValueType.field("key"); Types.NestedField valueField = keyValueType.field("value"); if (keyValueType.field("value").isOptional()) { return Types.MapType.ofOptional( keyField.fieldId(), valueField.fieldId(), keyField.type(), valueField.type()); } else { return Types.MapType.ofRequired( keyField.fieldId(), valueField.fieldId(), keyField.type(), valueField.type()); } } else { // normal array Schema elementSchema = array.getElementType(); int id = getElementId(array); if (AvroSchemaUtil.isOptionSchema(elementSchema)) { return Types.ListType.ofOptional(id, elementType); } else { return Types.ListType.ofRequired(id, elementType); } } }