private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType); } }
} else if (type.equals(Schema.Type.ARRAY)) { if (writeOldListStructure) { return ConversionPatterns.listType(repetition, fieldName, convertField("array", schema.getElementType(), REPEATED)); } else { return ConversionPatterns.listOfElements(repetition, fieldName, convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType())); Type valType = convertField("value", schema.getValueType()); return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType); } else if (type.equals(Schema.Type.FIXED)) { builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
/** * @param repetition repetition for the list * @param alias name of the field * @param nestedType type of elements in the list * @return a group representing the list using a 2-level representation * @deprecated use listOfElements instead */ @Deprecated public static GroupType listType(Repetition repetition, String alias, Type nestedType) { return listWrapper( repetition, alias, LIST, nestedType ); }
public static GroupType stringKeyMapType(Repetition repetition, String alias, Type valueType) { return stringKeyMapType(repetition, alias, "map", valueType); }
private ConvertedField visitListLike(ThriftField listLike, State state, boolean isSet) { State childState = new State(state.path, REPEATED, state.name + "_tuple"); ConvertedField converted = listLike.getType().accept(this, childState); if (converted.isKeep()) { // doProjection prevents an infinite recursion here if (isSet && doProjection) { ConvertedField fullConv = listLike .getType() .accept(new ThriftSchemaConvertVisitor(FieldProjectionFilter.ALL_COLUMNS, false, keepOneOfEachUnion), childState); if (!converted.asKeep().getType().equals(fullConv.asKeep().getType())) { throw new ThriftProjectionException("Cannot select only a subset of the fields in a set, " + "for path " + state.path); } } return new Keep(state.path, listType(state.repetition, state.name, converted.asKeep().getType())); } return new Drop(state.path); }
/** * @param repetition repetition for the list * @param alias name of the field * @param nestedType type of elements in the list * @return a group representing the list using a 2-level representation * @deprecated use listOfElements instead */ @Deprecated public static GroupType listType(Repetition repetition, String alias, Type nestedType) { return listWrapper( repetition, alias, LogicalTypeAnnotation.listType(), nestedType ); }
public static GroupType stringKeyMapType(Repetition repetition, String alias, Type valueType) { return stringKeyMapType(repetition, alias, "map", valueType); }
public static GroupType mapType(Repetition repetition, String alias, Type keyType, Type valueType) { return mapType(repetition, alias, "map", keyType, valueType); }
public static GroupType mapType(Repetition repetition, String alias, String mapAlias, Type keyType, Type valueType) { //support projection only on key of a map if (valueType == null) { return listWrapper( repetition, alias, MAP, new GroupType( Repetition.REPEATED, mapAlias, MAP_KEY_VALUE, keyType) ); } else { if (!valueType.getName().equals("value")) { throw new RuntimeException(valueType.getName() + " should be value"); } return listWrapper( repetition, alias, MAP, new GroupType( Repetition.REPEATED, mapAlias, MAP_KEY_VALUE, keyType, valueType) ); } }
public static GroupType mapType(Repetition repetition, String alias, Type keyType, Type valueType) { return mapType(repetition, alias, "map", keyType, valueType); }
/** * Creates a 3-level list structure annotated with LIST with elements of the * given elementType. The repeated level is inserted automatically and the * elementType's repetition should be the correct repetition of the elements, * required for non-null and optional for nullable. * * @param listRepetition the repetition of the entire list structure * @param name the name of the list structure type * @param elementType the type of elements contained by the list * @return a GroupType that represents the list */ public static GroupType listOfElements(Repetition listRepetition, String name, Type elementType) { Preconditions.checkArgument(elementType.getName().equals(ELEMENT_NAME), "List element type must be named 'element'"); return listWrapper( listRepetition, name, LIST, new GroupType(Repetition.REPEATED, "list", elementType) ); } }
public static GroupType stringKeyMapType(Repetition repetition, String alias, String mapAlias, Type valueType) { return mapType(repetition, alias, mapAlias, new PrimitiveType(Repetition.REQUIRED, PrimitiveTypeName.BINARY, "key", OriginalType.UTF8), valueType); }
/** * Creates a 3-level list structure annotated with LIST with elements of the * given elementType. The repeated level is inserted automatically and the * elementType's repetition should be the correct repetition of the elements, * required for non-null and optional for nullable. * * @param listRepetition the repetition of the entire list structure * @param name the name of the list structure type * @param elementType the type of elements contained by the list * @return a GroupType that represents the list */ public static GroupType listOfElements(Repetition listRepetition, String name, Type elementType) { Preconditions.checkArgument(elementType.getName().equals(ELEMENT_NAME), "List element type must be named 'element'"); return listWrapper( listRepetition, name, LogicalTypeAnnotation.listType(), new GroupType(Repetition.REPEATED, "list", elementType) ); } }
public static GroupType stringKeyMapType(Repetition repetition, String alias, String mapAlias, Type valueType) { return mapType(repetition, alias, mapAlias, new PrimitiveType(Repetition.REQUIRED, PrimitiveTypeName.BINARY, "key", stringType()), valueType); }
public static GroupType mapType(Repetition repetition, String alias, String mapAlias, Type keyType, Type valueType) { //support projection only on key of a map if (valueType == null) { return listWrapper( repetition, alias, LogicalTypeAnnotation.mapType(), new GroupType( Repetition.REPEATED, mapAlias, LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance(), keyType) ); } else { if (!valueType.getName().equals("value")) { throw new RuntimeException(valueType.getName() + " should be value"); } return listWrapper( repetition, alias, LogicalTypeAnnotation.mapType(), new GroupType( Repetition.REPEATED, mapAlias, LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance(), keyType, valueType) ); } }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo, final Repetition repetition) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(repetition, name, keyType, valueType); }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo, final Repetition repetition) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(repetition, name, keyType, valueType); }