/* package private */Schema stripOptionalTypeUnion(Schema schema) { if(schema.getType() == Schema.Type.UNION && schema.getTypes().size() == 2 && schema.getTypes().contains(NULL_TYPE_SCHEMA)) { return schema.getTypes().get(0).equals(NULL_TYPE_SCHEMA) ? schema.getTypes().get(1) : schema.getTypes().get(0); } return schema; }
private static List<Schema> getNonNullSubSchemas(final Schema avroSchema) { final List<Schema> unionFieldSchemas = avroSchema.getTypes(); if (unionFieldSchemas == null) { return Collections.emptyList(); } final List<Schema> nonNullTypes = new ArrayList<>(unionFieldSchemas.size()); for (final Schema fieldSchema : unionFieldSchemas) { if (fieldSchema.getType() != Type.NULL) { nonNullTypes.add(fieldSchema); } } return nonNullTypes; }
/** * Makes sure the default value is good * * @param parent * @param field */ /* package private */static void checkDefaultValueIsLegal(Field field, List<Message> messages, String name) { if(field == null) { throw new IllegalArgumentException("Field must be non-null. Name=" + name); } if(field.defaultValue() != null) { // Get the type schema. If this is a UNION, the default must be of // the leading type Schema fieldSchema = field.schema(); if(fieldSchema.getType() == Schema.Type.UNION) { fieldSchema = fieldSchema.getTypes().get(0); } // Get the default value JsonNode defaultJson = field.defaultValue(); String expectedVal = checkDefaultJson(defaultJson, field.schema()); if(expectedVal != null) { messages.add(new Message(Level.ERROR, "Illegal default value for field " + name + ". The default must be of type " + expectedVal + ".")); } } }
private Symbol resolveUnion(Schema writer, Schema reader, Map<LitS, Symbol> seen) throws IOException { boolean needsAdj = ! unionEquiv(writer, reader, new HashMap<>()); List<Schema> alts2 = (!needsAdj ? reader.getTypes() : null); List<Schema> alts = writer.getTypes(); final int size = alts.size(); Symbol[] symbols = new Symbol[size]; String[] labels = new String[size]; /** * We construct a symbol without filling the arrays. Please see * {@link Symbol#production} for the reason. */ int i = 0; for (Schema w : alts) { symbols[i] = generate(w, (needsAdj ? reader : alts2.get(i)), seen); labels[i] = w.getFullName(); i++; } if (! needsAdj) return Symbol.seq(Symbol.alt(symbols, labels), Symbol.UNION); return Symbol.seq(Symbol.alt(symbols, labels), Symbol.WRITER_UNION_ACTION); }
private Object deserializeUnion(Object datum, Schema fileSchema, Schema recordSchema, UnionTypeInfo columnType) throws AvroSerdeException { // Calculate tags individually since the schema can evolve and can have different tags. In worst case, both schemas are same // and we would end up doing calculations twice to get the same tag int fsTag = GenericData.get().resolveUnion(fileSchema, datum); // Determine index of value from fileSchema int rsTag = GenericData.get().resolveUnion(recordSchema, datum); // Determine index of value from recordSchema Object desered = worker(datum, fileSchema == null ? null : fileSchema.getTypes().get(fsTag), recordSchema.getTypes().get(rsTag), columnType.getAllUnionObjectTypeInfos().get(rsTag)); return new StandardUnionObjectInspector.StandardUnion((byte)rsTag, desered); }
for (Field field : target.getSchema().getFields()) { if (field.schema().getType() == Schema.Type.UNION) { if (actualFieldSchema.getType() == Schema.Type.RECORD) { for (Schema candidateType : field.schema().getTypes()) { if (candidateType.getFullName().equals(actualFieldSchema.getFullName())) { GenericRecord record = new GenericData.Record(candidateType); target.put(field.name(), source.get(field.name())); } else if (field.schema().getType() == Schema.Type.RECORD) { GenericRecord record = (GenericRecord) target.get(field.name()); if (record == null) {
/** * Gets the schema by full name. * * @param arraySchema the array schema * @param fullName the full name * @return the schema by full name */ private static Schema getSchemaByFullName(Schema arraySchema, String fullName) { if (arraySchema.getElementType().getType() == Schema.Type.UNION) { List<Schema> itemTypes = arraySchema.getElementType().getTypes(); return getSchemaByFullName(itemTypes, fullName); } else { return arraySchema.getElementType().getFullName().equals( fullName) ? arraySchema.getElementType() : null; } }
/** * Gets the delta schema by full name. * * @param fullName the full name * @return the delta schema by full name */ private Schema getDeltaSchemaByFullName(String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField(DELTA).schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
public static void fillComplexPartialDelta(GenericRecord delta) { GenericRecord testField22 = new GenericData.Record(getSchemaByFullName( delta.getSchema().getField("testField2").schema().getTypes(), "org.kaa.config.testRecordT")); testField22.put("testField3", 654); GenericEnumSymbol unchanged = new GenericData.EnumSymbol( getSchemaByFullName(delta.getSchema().getField("testField1") .schema().getTypes(), "org.kaaproject.configuration.unchangedT"), "unchanged"); delta.put("testField1", unchanged); delta.put("testField2", testField22); }
public static boolean isOptionalPrimitive(Schema schema) { return schema.getType().equals(Schema.Type.UNION) && schema.getTypes().size() == 2 && ( (schema.getTypes().get(0).getType().equals(Schema.Type.NULL) && (isPrimitive(schema.getTypes().get(1)) || isPrimitiveArray(schema.getTypes().get(1)))) || (schema.getTypes().get(1).getType().equals(Schema.Type.NULL) && (isPrimitive(schema.getTypes().get(0)) || isPrimitiveArray(schema.getTypes().get(0)))) ); }
private static String extractAvroTypeFromUnion(Schema.Field field) { if (field.schema().getTypes().size() >= 3) { LOG.warn("Avro schema field " + field.name() + " has 3 or more types: using the first non-null type"); } for (Schema schema : field.schema().getTypes()) { if (!schema.getType().toString().equalsIgnoreCase("NULL")) { return schema.getType().toString(); } } String message = "Avro schema field " + field.name() + " is a union, but it does not contain a non-null field type."; LOG.error(message); throw new RuntimeException(message); }
@Test public void shouldSupportAvroStructs() { // When: final org.apache.avro.Schema avroSchema = SchemaUtil.buildAvroSchema(schema, "bob"); // Then: final org.apache.avro.Schema.Field rawStruct = avroSchema.getField("RAW_STRUCT"); assertThat(rawStruct, is(notNullValue())); assertThat(rawStruct.schema().getType(), is(org.apache.avro.Schema.Type.UNION)); assertThat(rawStruct.schema().getTypes().get(0).getType(), is(org.apache.avro.Schema.Type.NULL)); assertThat(rawStruct.schema().getTypes().get(1).toString(), is( "{" + "\"type\":\"record\"," + "\"name\":\"RAW_STRUCT\"," + "\"namespace\":\"ksql.bob\"," + "\"fields\":[" + "{\"name\":\"f0\",\"type\":[\"null\",\"long\"],\"default\":null}," + "{\"name\":\"f1\",\"type\":[\"null\",\"boolean\"],\"default\":null}" + "]}" )); }
@Override void toJson1(JsonGenerator gen) throws IOException { gen.writeFieldName("response"); response.toJson(types, gen); List<Schema> errs = errors.getTypes(); // elide system error if (errs.size() > 1) { Schema union = Schema.createUnion(errs.subList(1, errs.size())); gen.writeFieldName("errors"); union.toJson(types, gen); } }
private Object serializeUnion(UnionTypeInfo typeInfo, UnionObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { byte tag = fieldOI.getTag(structFieldData); // Invariant that Avro's tag ordering must match Hive's. return serialize(typeInfo.getAllUnionObjectTypeInfos().get(tag), fieldOI.getObjectInspectors().get(tag), fieldOI.getField(structFieldData), schema.getTypes().get(tag)); }
/** * Returns true if this field is optional. Optional fields are represented * as a type union containing the null type. * * @param field * @return */ /* package private */boolean isOptional(Field field) { if(field.schema().getType() == Type.UNION) { for(Schema nestedType: field.schema().getTypes()) { if(nestedType.getType() == Type.NULL) { return true; } } } return false; }
private static Schema getDeltaSchemaByFullName(Schema deltaSchema, String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField("delta").schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
/** Utility for template use. For a two-branch union type with * one null branch, returns the index of the null branch. It's an * error to use on anything other than a two-branch union with on * null branch. */ public int getNonNullIndex(Schema s) { if (s.getType() != Schema.Type.UNION || s.getTypes().size() != 2 || ! s.getTypes().contains(NULL_SCHEMA)) throw new IllegalArgumentException("Can only be used on 2-branch union with a null branch: " + s); return (s.getTypes().get(0).equals(NULL_SCHEMA) ? 1 : 0); }