/** * Return whether the Avro field is a single-value field. */ public static boolean isSingleValueField(Field field) { try { org.apache.avro.Schema fieldSchema = extractSupportedSchema(field.schema()); return fieldSchema.getType() != org.apache.avro.Schema.Type.ARRAY; } catch (Exception e) { throw new RuntimeException("Caught exception while extracting non-null schema from field: " + field.name(), e); } }
GenericRecord avroRecord = new GenericData.Record(outputSchema); JsonElementConversionWithAvroSchemaFactory.JsonElementConverter converter; for (Schema.Field field : outputSchema.getFields()) { if (ignoreFields.contains(field.name())) { continue; Schema.Type type = field.schema().getType(); boolean nullable = false; Schema schema = field.schema(); if (inputRecord.get(field.name()) == null) { inputRecord.add(field.name(), JsonNull.INSTANCE); if (inputRecord.get(field.name()) == null) { throw new DataConversionException("Field missing from record: " + field.name()); if (nullable && inputRecord.get(field.name()).isJsonNull()) { avroRecord.put(field.name(), null); } else { avroRecord.put(field.name(), convertNestedRecord(schema, inputRecord.get(field.name()).getAsJsonObject(), workUnit, ignoreFields)); converter = JsonElementConversionWithAvroSchemaFactory.getConvertor(field.name(), type.getName(), schema, workUnit, nullable, ignoreFields); avroRecord.put(field.name(), converter.convert(inputRecord.get(field.name()))); } catch (Exception e) { throw new DataConversionException("Could not convert field " + field.name(), e);
private static TypeInfo generateRecordTypeInfo(Schema schema, Set<Schema> seenSchemas) throws AvroSerdeException { assert schema.getType().equals(Schema.Type.RECORD); if (seenSchemas == null) { seenSchemas = Collections.newSetFromMap(new IdentityHashMap<Schema, Boolean>()); } else if (seenSchemas.contains(schema)) { throw new AvroSerdeException( "Recursive schemas are not supported. Recursive schema was " + schema .getFullName()); } seenSchemas.add(schema); List<Schema.Field> fields = schema.getFields(); List<String> fieldNames = new ArrayList<String>(fields.size()); List<TypeInfo> typeInfos = new ArrayList<TypeInfo>(fields.size()); for(int i = 0; i < fields.size(); i++) { fieldNames.add(i, fields.get(i).name()); typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas)); } return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); }
private static String extractAvroTypeFromUnion(Schema.Field field) { if (field.schema().getTypes().size() >= 3) { LOG.warn("Avro schema field " + field.name() + " has 3 or more types: using the first non-null type"); } for (Schema schema : field.schema().getTypes()) { if (!schema.getType().toString().equalsIgnoreCase("NULL")) { return schema.getType().toString(); } } String message = "Avro schema field " + field.name() + " is a union, but it does not contain a non-null field type."; LOG.error(message); throw new RuntimeException(message); }
assert (writerSchema.getType() == Type.RECORD); final List<Field> writerFields = new ArrayList<Field>(); final Field direct = writerSchema.getField(readerField.name()); if (direct != null) { writerFields.add(direct); switch (writerFields.size()) { case 0: return null; case 1: return writerFields.get(0);
private static JsonNode injectUuidsFromJsonNodes(JsonNode json, Schema schema) { if (json == null) { return json; } switch (schema.getType()) { case RECORD: schema.getFields().stream() .filter(f -> !f.name().equals(UUID_FIELD)) .forEach(f -> injectUuidsFromJsonNodes(json.get(f.name()), f.schema())); boolean addressable = schema.getFields().stream().filter(f -> f.name().equals( UUID_FIELD)).findFirst().isPresent(); if (addressable) { ((ObjectNode) json).put(UUID_FIELD, (Integer) null); } break; case UNION: schema.getTypes() .forEach(s -> injectUuidsFromJsonNodes(json.get(s.getName()), s)); break; case ARRAY: json.getElements().forEachRemaining((elem) -> injectUuids(elem, schema.getElementType())); break; default: return json; } return json; }
private List<Schema.Field> getFields(Schema.Field schemaField) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); JsonNode nullDefault = JsonNodeFactory.instance.nullNode(); if (schemaField.schema().getType() == Schema.Type.RECORD) { for (Schema.Field field : schemaField.schema().getFields()) { fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault)); } } else { fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(), nullDefault)); } return fields; }
/** * Convert to the output schema of a field */ protected Field convertFieldSchema(Schema inputSchema, Field field, WorkUnitState workUnit) throws SchemaConversionException { if (field.name().equals(payloadField)) { // Create a payload field with latest schema return createLatestPayloadField(field); } // Make a copy of the field to the output schema return new Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order()); }
private static void _copyFields( List<Schema.Field> fields, List<Schema.Field> copiedFieldList) { for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field( field.name(), field.schema(), field.doc(), field.defaultVal()); _copyFieldProperties(field.getObjectProps(), newField); copiedFieldList.add(newField); } }
/** Called to read a single field of a record. May be overridden for more * efficient or alternate implementations.*/ protected void readField(Object r, Field f, Object oldDatum, ResolvingDecoder in, Object state) throws IOException { data.setField(r, f.name(), f.pos(), read(oldDatum, f.schema(), in), state); }
/** Called to write a single field of a record. May be overridden for more * efficient or alternate implementations.*/ protected void writeField(Object datum, Field f, Encoder out, Object state) throws IOException { Object value = data.getField(datum, f.name(), f.pos(), state); try { write(f.schema(), value, out); } catch (NullPointerException e) { throw npe(e, " in field " + f.name()); } }
/** Called to write a single field of a record. May be overridden for more * efficient or alternate implementations.*/ protected void writeField(Object datum, Field f, Encoder out, Object state) throws IOException { Object value = data.getField(datum, f.name(), f.pos(), state); try { write(f.schema(), value, out); } catch (NullPointerException e) { throw npe(e, " in field " + f.name()); } }
/** Called to read a single field of a record. May be overridden for more * efficient or alternate implementations.*/ protected void readField(Object r, Field f, Object oldDatum, ResolvingDecoder in, Object state) throws IOException { data.setField(r, f.name(), f.pos(), read(oldDatum, f.schema(), in), state); }
/** * Process field of a record type. * * @param fieldDefinition schema for field. * @return generated value for field based on its definition. * @throws ConfigurationGenerationException configuration processing exception */ private Object processField(Field fieldDefinition) throws ConfigurationGenerationException { // if this a "uuid" type then generate it if (UUID_FIELD.equals(fieldDefinition.name())) { return AvroUtils.generateUuidObject(); } return processType(fieldDefinition.schema(), fieldDefinition.getJsonProp(BY_DEFAULT_FIELD)); }
private static List<Schema.Field> _cloneFieldsAndResetPosition( List<Schema.Field> fields) { List<Schema.Field> copyFieldList = new ArrayList<>(); for (Schema.Field schemaEntry : fields) { Schema.Field field = new Schema.Field( schemaEntry.name(), schemaEntry.schema(), schemaEntry.doc(), schemaEntry.defaultVal(), schemaEntry.order()); Map<String, Object> objectProperties = schemaEntry.getObjectProps(); for (Map.Entry<String, Object> entry : objectProperties.entrySet()) { field.addProp(entry.getKey(), entry.getValue()); } copyFieldList.add(field); } return copyFieldList; }
@Test public void testToStringIsJson() throws JsonParseException, IOException { Field stringField = new Field("string", Schema.create(Type.STRING), null, null); Field enumField = new Field("enum", Schema.createEnum("my_enum", "doc", null, Arrays.asList("a", "b", "c")), null, null); Schema schema = Schema.createRecord("my_record", "doc", "mytest", false); schema.setFields(Arrays.asList(stringField, enumField)); GenericRecord r = new GenericData.Record(schema); // \u2013 is EN DASH r.put(stringField.name(), "hello\nthere\"\tyou\u2013}"); r.put(enumField.name(), new GenericData.EnumSymbol(enumField.schema(),"a")); String json = r.toString(); JsonFactory factory = new JsonFactory(); JsonParser parser = factory.createJsonParser(json); ObjectMapper mapper = new ObjectMapper(); // will throw exception if string is not parsable json mapper.readTree(parser); }
@Override protected void writeField(Object datum, Schema.Field f, Encoder out, Object state) throws IOException { if (datum instanceof SpecificRecordBase) { Conversion<?> conversion = ((SpecificRecordBase) datum).getConversion(f.pos()); Schema fieldSchema = f.schema(); LogicalType logicalType = fieldSchema.getLogicalType(); Object value = getData().getField(datum, f.name(), f.pos()); if (conversion != null && logicalType != null) { value = convert(fieldSchema, logicalType, conversion, value); } writeWithoutConversion(fieldSchema, value, out); } else { super.writeField(datum, f, out, state); } } }