public AvroRecordMaterializer(MessageType requestedSchema, Schema avroSchema, GenericData baseModel) { this.root = new AvroRecordConverter<T>(requestedSchema, avroSchema, baseModel); }
private void fillInDefaults() { for (Map.Entry<Schema.Field, Object> entry : recordDefaults.entrySet()) { Schema.Field f = entry.getKey(); // replace following with model.deepCopy once AVRO-1455 is being used Object defaultValue = deepCopy(f.schema(), entry.getValue()); set(f.name(), f.pos(), defaultValue); } }
@Override public T getCurrentRecord() { return root.getCurrentRecord(); }
recordClass = getDatumClass(avroSchema, model); Map<String, Class<?>> fields = getFieldsByName(recordClass, false); final Schema.Field avroField = getAvroField(parquetField.getName()); Schema nonNullSchema = AvroSchemaConverter.getNonNull(avroField.schema()); final int finalAvroIndex = avroFieldIndexes.remove(avroField.name()); converters[parquetFieldIndex] = newConverter( nonNullSchema, parquetField, this.model, fieldClass, container);
return new AvroConverters.FieldBooleanConverter(parent); } else if (schema.getType().equals(Schema.Type.INT)) { Class<?> datumClass = getDatumClass(conversion, knownClass, schema, model); if (datumClass == null) { return new AvroConverters.FieldIntegerConverter(parent); return new AvroConverters.FieldDoubleConverter(parent); } else if (schema.getType().equals(Schema.Type.BYTES)) { Class<?> datumClass = getDatumClass(conversion, knownClass, schema, model); if (datumClass == null) { return new AvroConverters.FieldByteBufferConverter(parent); return newStringConverter(schema, model, parent); } else if (schema.getType().equals(Schema.Type.RECORD)) { return new AvroRecordConverter(parent, type.asGroupType(), schema, model); } else if (schema.getType().equals(Schema.Type.ENUM)) { return new AvroConverters.FieldEnumConverter(parent, schema, model); } else if (schema.getType().equals(Schema.Type.ARRAY)) { Class<?> datumClass = getDatumClass(conversion, knownClass, schema, model); if (datumClass != null && datumClass.isArray()) { return new AvroArrayConverter(
public AvroArrayConverter(ParentValueContainer parent, GroupType type, Schema avroSchema, GenericData model, Class<?> arrayClass) { this.parent = parent; this.avroSchema = avroSchema; Preconditions.checkArgument(arrayClass.isArray(), "Cannot convert non-array: " + arrayClass.getName()); this.elementClass = arrayClass.getComponentType(); ParentValueContainer setter = createSetterAndContainer(); Schema elementSchema = this.avroSchema.getElementType(); Type repeatedType = type.getType(0); // always determine whether the repeated type is the element type by // matching it against the element schema. if (isElementType(repeatedType, elementSchema)) { // the element type is the repeated type (and required) converter = newConverter(elementSchema, repeatedType, model, elementClass, setter); } else { // the element is wrapped in a synthetic group and may be optional converter = new ArrayElementConverter( repeatedType.asGroupType(), elementSchema, model, setter); } }
private static <T> Class<T> getDatumClass(Schema schema, GenericData model) { return getDatumClass(null, null, schema, model); }
@Override public void end() { fillInDefaults(); if (parent != null) { parent.add(currentRecord); } else { // this applies any converters needed for the root value rootContainer.add(currentRecord); } }
private static Map<String, Class<?>> getFieldsByName(Class<?> recordClass, boolean excludeJava) { Map<String, Class<?>> fields = new LinkedHashMap<String, Class<?>>(); if (recordClass != null) { Class<?> current = recordClass; do { if (excludeJava && current.getPackage() != null && current.getPackage().getName().startsWith("java.")) { break; // skip java built-in classes } for (Field field : current.getDeclaredFields()) { if (field.isAnnotationPresent(AvroIgnore.class) || isTransientOrStatic(field)) { continue; } AvroName altName = field.getAnnotation(AvroName.class); Class<?> existing = fields.put( altName != null ? altName.value() : field.getName(), field.getType()); if (existing != null) { throw new AvroTypeException( current + " contains two fields named: " + field.getName()); } } current = current.getSuperclass(); } while (current != null); } return fields; }
private static Converter newStringConverter(Schema schema, GenericData model, ParentValueContainer parent) { Class<?> stringableClass = getStringableClass(schema, model); if (stringableClass == String.class) { return new FieldStringConverter(parent); } else if (stringableClass == CharSequence.class) { return new AvroConverters.FieldUTF8Converter(parent); } return new FieldStringableConverter(parent, stringableClass); }
public AvroArrayConverter(ParentValueContainer parent, GroupType type, Schema avroSchema, GenericData model) { this.parent = parent; this.avroSchema = avroSchema; Schema elementSchema = AvroSchemaConverter .getNonNull(avroSchema.getElementType()); Type repeatedType = type.getType(0); // always determine whether the repeated type is the element type by // matching it against the element schema. if (AvroRecordConverter.isElementType(repeatedType, elementSchema)) { // the element type is the repeated type (and required) converter = newConverter(elementSchema, repeatedType, model, new ParentValueContainer() { @Override @SuppressWarnings("unchecked") public void add(Object value) { array.add(value); } }); } else { // the element is wrapped in a synthetic group and may be optional converter = new ElementConverter(repeatedType.asGroupType(), elementSchema, model); } }
public AvroCollectionConverter(ParentValueContainer parent, GroupType type, Schema avroSchema, GenericData model, Class<?> containerClass) { this.parent = parent; this.avroSchema = avroSchema; this.containerClass = containerClass; Schema elementSchema = AvroSchemaConverter.getNonNull(avroSchema.getElementType()); Type repeatedType = type.getType(0); // always determine whether the repeated type is the element type by // matching it against the element schema. if (isElementType(repeatedType, elementSchema)) { // the element type is the repeated type (and required) converter = newConverter(elementSchema, repeatedType, model, new ParentValueContainer() { @Override @SuppressWarnings("unchecked") public void add(Object value) { container.add(value); } }); } else { // the element is wrapped in a synthetic group and may be optional converter = new ElementConverter(repeatedType.asGroupType(), elementSchema, model); } }