private Type convertField(String fieldName, Schema schema) { return convertField(fieldName, schema, Type.Repetition.REQUIRED); }
return Schema.createArray(convertField(repeatedType)); } else { Type elementType = repeatedType.asGroupType().getType(0); if (elementType.isRepetition(Type.Repetition.OPTIONAL)) { return Schema.createArray(optional(convertField(elementType))); } else { return Schema.createArray(convertField(elementType)); return Schema.createMap(optional(convertField(valueType))); } else { return Schema.createMap(convertField(valueType));
if (writeOldListStructure) { return ConversionPatterns.listType(repetition, fieldName, convertField("array", schema.getElementType(), REPEATED)); } else { return ConversionPatterns.listOfElements(repetition, fieldName, convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType())); Type valType = convertField("value", schema.getValueType());
private Type convertField(Schema.Field field) { return convertField(field.name(), field.schema()); }
private List<Type> convertFields(List<Schema.Field> fields) { List<Type> types = new ArrayList<Type>(); for (Schema.Field field : fields) { if (field.schema().getType().equals(Schema.Type.NULL)) { continue; // Avro nulls are not encoded, unless they are null unions } types.add(convertField(field)); } return types; }
private Type convertUnion(String fieldName, Schema schema, Type.Repetition repetition) { List<Schema> nonNullSchemas = new ArrayList<Schema>(schema.getTypes().size()); for (Schema childSchema : schema.getTypes()) { if (childSchema.getType().equals(Schema.Type.NULL)) { if (Type.Repetition.REQUIRED == repetition) { repetition = Type.Repetition.OPTIONAL; } } else { nonNullSchemas.add(childSchema); } } // If we only get a null and one other type then its a simple optional field // otherwise construct a union container switch (nonNullSchemas.size()) { case 0: throw new UnsupportedOperationException("Cannot convert Avro union of only nulls"); case 1: return convertField(fieldName, nonNullSchemas.get(0), repetition); default: // complex union type List<Type> unionTypes = new ArrayList<Type>(nonNullSchemas.size()); int index = 0; for (Schema childSchema : nonNullSchemas) { unionTypes.add( convertField("member" + index++, childSchema, Type.Repetition.OPTIONAL)); } return new GroupType(repetition, fieldName, unionTypes); } }
private Schema convertFields(String name, List<Type> parquetFields) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); for (Type parquetType : parquetFields) { Schema fieldSchema = convertField(parquetType); if (parquetType.isRepetition(REPEATED)) { throw new UnsupportedOperationException("REPEATED not supported outside LIST or MAP. Type: " + parquetType); } else if (parquetType.isRepetition(Type.Repetition.OPTIONAL)) { fields.add(new Schema.Field( parquetType.getName(), optional(fieldSchema), null, NULL_VALUE)); } else { // REQUIRED fields.add(new Schema.Field( parquetType.getName(), fieldSchema, null, (Object) null)); } } Schema schema = Schema.createRecord(name, null, null, false); schema.setFields(fields); return schema; }