private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo, final Repetition repetition) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(repetition, name, keyType, valueType); }
/** * @param repetition * @param alias name of the field * @param nestedType * @return */ public static GroupType listType(Repetition repetition, String alias, Type nestedType) { return listWrapper( repetition, alias, LIST, nestedType ); } }
@Override public void visit(ThriftType.SetType setType) { final ThriftField setElemField = setType.getValues(); String setName = currentName; Type.Repetition setRepetition = currentRepetition; currentName = currentName + "_tuple"; currentRepetition = REPEATED; setElemField.getType().accept(this); //after conversion, currentType is the nested type if (currentType == null) { return; } else { currentType = listType(setRepetition, setName, currentType); } }
public static GroupType stringKeyMapType(Repetition repetition, String alias, Type valueType) { return stringKeyMapType(repetition, alias, "map", valueType); }
public static GroupType mapType(Repetition repetition, String alias, String mapAlias, Type keyType, Type valueType) { //support projection only on key of a map if (valueType == null) { return listWrapper( repetition, alias, MAP, new GroupType( Repetition.REPEATED, mapAlias, MAP_KEY_VALUE, keyType) ); } else { if (!valueType.getName().equals("value")) { throw new RuntimeException(valueType.getName() + " should be value"); } return listWrapper( repetition, alias, MAP, new GroupType( Repetition.REPEATED, mapAlias, MAP_KEY_VALUE, keyType, valueType) ); } }
@Override public void visit(ThriftType.ListType listType) { final ThriftField setElemField = listType.getValues(); String listName = currentName; Type.Repetition listRepetition = currentRepetition; currentName = currentName + "_tuple"; currentRepetition = REPEATED; setElemField.getType().accept(this); //after conversion, currentType is the nested type if (currentType == null) { return; } else { currentType = listType(listRepetition, listName, currentType); } }
/** * * @param alias * @param fieldSchema * @return an optional group containing one repeated group field (key, value) * @throws FrontendException */ private GroupType convertMap(String alias, FieldSchema fieldSchema) { Schema innerSchema = fieldSchema.schema; if (innerSchema == null || innerSchema.size() != 1) { throw new SchemaConversionException("Invalid map Schema, schema should contain exactly one field: " + fieldSchema); } FieldSchema innerField = null; try { innerField = innerSchema.getField(0); } catch (FrontendException fe) { throw new SchemaConversionException("Invalid map schema, cannot infer innerschema: ", fe); } Type convertedValue = convertWithName(innerField, "value"); return ConversionPatterns.stringKeyMapType(Repetition.OPTIONAL, alias, name(innerField.alias, "map"), convertedValue); }
public static GroupType mapType(Repetition repetition, String alias, Type keyType, Type valueType) { return mapType(repetition, alias, "map", keyType, valueType); }
public static GroupType mapType(Repetition repetition, String alias, Type valueType) { if (!valueType.getName().equals("value")) { throw new RuntimeException(valueType.getName() + " should be value"); } return listWrapper( repetition, alias, MAP, new GroupType( Repetition.REPEATED, "map", MAP_KEY_VALUE, new PrimitiveType(Repetition.REQUIRED, PrimitiveTypeName.BINARY, "key"), valueType) ); }
/** * * @param name * @param fieldSchema * @return an optional group containing one repeated group field * @throws FrontendException */ private GroupType convertBag(String name, FieldSchema fieldSchema) throws FrontendException { FieldSchema innerField = fieldSchema.schema.getField(0); return ConversionPatterns.listType( Repetition.OPTIONAL, name, convertTuple(name(innerField.alias, "bag"), innerField, Repetition.REPEATED)); }
public static GroupType stringKeyMapType(Repetition repetition, String alias, String mapAlias, Type valueType) { return mapType(repetition, alias, mapAlias, new PrimitiveType(Repetition.REQUIRED, PrimitiveTypeName.BINARY, "key", OriginalType.UTF8), valueType); }
public static GroupType listType(Repetition repetition, String alias, Type nestedType) { GroupType repeatedField; if (nestedType.isPrimitive()) { repeatedField = new GroupType( Repetition.REPEATED, "bag", nestedType); } else { final GroupType nestedGroupType = nestedType.asGroupType(); repeatedField = new GroupType( Repetition.REPEATED, nestedGroupType.getName(), nestedGroupType.getFields()); } return listWrapper( repetition, alias, LIST, repeatedField ); } }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType); }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo, final Repetition repetition) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(repetition, name, keyType, valueType); }
currentType = mapType(currentRepetition, currentName, keyType, valueType);