private static boolean isWritableType(TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); return isWritablePrimitiveType(primitiveCategory); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; return isWritableType(mapTypeInfo.getMapKeyTypeInfo()) && isWritableType(mapTypeInfo.getMapValueTypeInfo()); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; return isWritableType(listTypeInfo.getListElementTypeInfo()); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; return structTypeInfo.getAllStructFieldTypeInfos().stream().allMatch(HiveWriteUtils::isWritableType); } return false; }
public static TypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { ArrayList<TypeInfo> signature = new ArrayList<TypeInfo>(2); signature.add(keyTypeInfo); signature.add(valueTypeInfo); TypeInfo result = cachedMapTypeInfo.get(signature); if (result == null) { result = new MapTypeInfo(keyTypeInfo, valueTypeInfo); TypeInfo prev = cachedMapTypeInfo.putIfAbsent(signature, result); if (prev != null) { result = prev; } } return result; }
private Object serializeMap(MapTypeInfo typeInfo, MapObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { // Avro only allows maps with string keys if(!mapHasStringKey(fieldOI.getMapKeyObjectInspector())) { throw new AvroSerdeException("Avro only supports maps with keys as Strings. Current Map is: " + typeInfo.toString()); } ObjectInspector mapKeyObjectInspector = fieldOI.getMapKeyObjectInspector(); ObjectInspector mapValueObjectInspector = fieldOI.getMapValueObjectInspector(); TypeInfo mapKeyTypeInfo = typeInfo.getMapKeyTypeInfo(); TypeInfo mapValueTypeInfo = typeInfo.getMapValueTypeInfo(); Map<?,?> map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); Map<Object, Object> deserialized = new LinkedHashMap<Object, Object>(fieldOI.getMapSize(structFieldData)); for (Map.Entry<?, ?> entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), STRING_SCHEMA), serialize(mapValueTypeInfo, mapValueObjectInspector, entry.getValue(), valueType)); } return deserialized; }
@Override public boolean equals(Object other) { if (this == other) { return true; } if (!(other instanceof MapTypeInfo)) { return false; } MapTypeInfo o = (MapTypeInfo) other; return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); }
static ListTypeInfo toStructListTypeInfo(MapTypeInfo mapTypeInfo) { final StructTypeInfo structTypeInfo = new StructTypeInfo(); structTypeInfo.setAllStructFieldNames(Lists.newArrayList("keys", "values")); structTypeInfo.setAllStructFieldTypeInfos(Lists.newArrayList( mapTypeInfo.getMapKeyTypeInfo(), mapTypeInfo.getMapValueTypeInfo())); final ListTypeInfo structListTypeInfo = new ListTypeInfo(); structListTypeInfo.setListElementTypeInfo(structTypeInfo); return structListTypeInfo; }
public static boolean lazyCompareMap(MapTypeInfo mapTypeInfo, Map<Object, Object> map, Map<Object, Object> expectedMap) { TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo(); TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); if (map.size() != expectedMap.size()) { throw new RuntimeException("Map key/value deserialized map.size() " + map.size() + " map " + map.toString() + " expectedMap.size() " + expectedMap.size() + " expectedMap " + expectedMap.toString() + " does not match keyTypeInfo " + keyTypeInfo.toString() + " valueTypeInfo " + valueTypeInfo.toString()); } return true; }
if (argumentAccepted.equals(argumentPassed) || TypeInfoUtils.doPrimitiveCategoriesMatch(argumentPassed, argumentAccepted)) { if (argumentPassed.equals(TypeInfoFactory.voidTypeInfo)) { if (argumentPassed.getCategory().equals(Category.LIST) && argumentAccepted.getCategory().equals(Category.LIST)) { .getListElementTypeInfo(); TypeInfo argumentAcceptedElement = ((ListTypeInfo) argumentAccepted) .getListElementTypeInfo(); return matchCost(argumentPassedElement, argumentAcceptedElement, exact); .getMapKeyTypeInfo(); TypeInfo argumentAcceptedKey = ((MapTypeInfo) argumentAccepted) .getMapKeyTypeInfo(); TypeInfo argumentPassedValue = ((MapTypeInfo) argumentPassed) .getMapValueTypeInfo(); TypeInfo argumentAcceptedValue = ((MapTypeInfo) argumentAccepted) .getMapValueTypeInfo(); int cost1 = matchCost(argumentPassedKey, argumentAcceptedKey, exact); int cost2 = matchCost(argumentPassedValue, argumentAcceptedValue, exact);
public static TypeDescription convertTypeInfo(TypeInfo info) { switch (info.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo) info; switch (pinfo.getPrimitiveCategory()) { case BOOLEAN: return TypeDescription.createBoolean(); " category " + pinfo.getPrimitiveCategory()); ListTypeInfo linfo = (ListTypeInfo) info; return TypeDescription.createList (convertTypeInfo(linfo.getListElementTypeInfo())); (convertTypeInfo(minfo.getMapKeyTypeInfo()), convertTypeInfo(minfo.getMapValueTypeInfo())); for (TypeInfo child: minfo.getAllUnionObjectTypeInfos()) { result.addUnionChild(convertTypeInfo(child)); StructTypeInfo sinfo = (StructTypeInfo) info; TypeDescription result = TypeDescription.createStruct(); for(String fieldName: sinfo.getAllStructFieldNames()) { result.addField(fieldName, convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName))); info.getCategory());
private static Field toField(String name, TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: final PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; switch (primitiveTypeInfo.getPrimitiveCategory()) { case BOOLEAN: return Field.nullable(name, MinorType.BIT.getType()); case UNION: final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); final List<Field> unionFields = Lists.newArrayList(); final int unionSize = unionFields.size(); case MAP: final MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; final TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo(); final TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); final StructTypeInfo mapStructTypeInfo = new StructTypeInfo(); Lists.newArrayList(keyTypeInfo, valueTypeInfo)); final ListTypeInfo mapListStructTypeInfo = new ListTypeInfo(); mapListStructTypeInfo.setListElementTypeInfo(mapStructTypeInfo);
List<TypeInfo> unionTypeInfos = ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos(); while (index < unionTypeInfos.size() && !unionTypeInfos.get(index).equals(objectTypeInfo)) { index++; GenericData.Array array = ((GenericData.Array) o); TypeInfo listTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo(); return array.stream().map((element) -> convertToORCObject(listTypeInfo, element)).collect(Collectors.toList()); TypeInfo keyInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); TypeInfo valueInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); throw new IllegalArgumentException("Error converting object of type " + o.getClass().getName() + " to ORC type " + typeInfo.getTypeName()); } else { return null;
private static int hashOfMap(MapTypeInfo type, Block singleMapBlock) { TypeInfo keyTypeInfo = type.getMapKeyTypeInfo(); TypeInfo valueTypeInfo = type.getMapValueTypeInfo(); int result = 0; for (int i = 0; i < singleMapBlock.getPositionCount(); i += 2) { result += hash(keyTypeInfo, singleMapBlock, i) ^ hash(valueTypeInfo, singleMapBlock, i + 1); } return result; }
boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); if (isList) { objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); if (objectTypeInfo.getCategory() != Category.STRUCT) { throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); if (isList) { t = TypeInfoFactory.getListTypeInfo(t); if (myt.getCategory() == Category.LIST) { TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else if (myt.getCategory() == Category.MAP) { if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), ((MapTypeInfo) myt).getMapKeyTypeInfo())) { throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE .getMsg(expr)); TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else {
boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); if (isList) { objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); if (objectTypeInfo.getCategory() != Category.STRUCT) { throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); if (isList) { t = TypeInfoFactory.getListTypeInfo(t); if (myt.getCategory() == Category.LIST) { TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else if (myt.getCategory() == Category.MAP) { if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), ((MapTypeInfo) myt).getMapKeyTypeInfo())) { throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE .getMsg(expr)); TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else {
public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); this.toType = requireNonNull(toHiveType, "toHiveType is null").getType(typeManager); HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); this.keyCoercer = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType); this.valueCoercer = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType); }
private static void validateAvroType(TypeInfo type, String columnName) { if (type.getCategory() == Category.MAP) { TypeInfo keyType = mapTypeInfo(type).getMapKeyTypeInfo(); if ((keyType.getCategory() != Category.PRIMITIVE) || (primitiveTypeInfo(keyType).getPrimitiveCategory() != PrimitiveCategory.STRING)) { throw new PrestoException(NOT_SUPPORTED, format("Column %s has a non-varchar map key, which is not supported by Avro", columnName)); } } else if (type.getCategory() == Category.PRIMITIVE) { PrimitiveCategory primitive = primitiveTypeInfo(type).getPrimitiveCategory(); if (primitive == PrimitiveCategory.BYTE) { throw new PrestoException(NOT_SUPPORTED, format("Column %s is tinyint, which is not supported by Avro. Use integer instead.", columnName)); } if (primitive == PrimitiveCategory.SHORT) { throw new PrestoException(NOT_SUPPORTED, format("Column %s is smallint, which is not supported by Avro. Use integer instead.", columnName)); } } }
private TypeInfo getFieldTypeIgnoreCase(TypeInfo hiveTypeInfo, String fieldName, int fieldIndex) { if (hiveTypeInfo == null) { return null; } else if (hiveTypeInfo.getCategory().equals(ObjectInspector.Category.STRUCT)) { return getStructFieldTypeInfo(fieldName, fieldIndex); } else if (hiveTypeInfo.getCategory().equals(ObjectInspector.Category.MAP)) { //This cover the case where hive table may have map<key, value> but the data file is // of type array<struct<value1, value2>> //Using index in place of type name. if (fieldIndex == 0) { return ((MapTypeInfo) hiveTypeInfo).getMapKeyTypeInfo(); } else if (fieldIndex == 1) { return ((MapTypeInfo) hiveTypeInfo).getMapValueTypeInfo(); } else {//Other fields are skipped for this case return null; } } throw new RuntimeException("Unknown hive type info " + hiveTypeInfo + " when searching for field " + fieldName); }
if (colMap.qualifierName == null && !colMap.hbaseRowKey && !colMap.hbaseTimestamp) { TypeInfo typeInfo = columnTypes.get(i); if ((typeInfo.getCategory() != ObjectInspector.Category.MAP) || (((MapTypeInfo) typeInfo).getMapKeyTypeInfo().getCategory() != ObjectInspector.Category.PRIMITIVE)) { + "' should be mapped to Map<? extends LazyPrimitive<?, ?>,?>, that is " + "the Key for the map should be of primitive type, but is mapped to " + typeInfo.getTypeName());
private Schema createAvroMap(TypeInfo typeInfo) { TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); if (((PrimitiveTypeInfo) keyTypeInfo).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING) { throw new UnsupportedOperationException("Key of Map can only be a String"); } TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); Schema valueSchema = createAvroSchema(valueTypeInfo); return Schema.createMap(valueSchema); }
((ListTypeInfo) hiveType).getListElementTypeInfo() : (hiveType instanceof MapTypeInfo) ? ((MapTypeInfo) hiveType).getMapValueTypeInfo() : hiveType; String typeName = TypeInfoUtils.getBaseName(realHiveType.getTypeName());
@Override public void readBatch(int total, ColumnVector column, TypeInfo columnType) throws IOException { MapColumnVector mapColumnVector = (MapColumnVector) column; MapTypeInfo mapTypeInfo = (MapTypeInfo) columnType; ListTypeInfo keyListTypeInfo = new ListTypeInfo(); keyListTypeInfo.setListElementTypeInfo(mapTypeInfo.getMapKeyTypeInfo()); ListTypeInfo valueListTypeInfo = new ListTypeInfo(); valueListTypeInfo.setListElementTypeInfo(mapTypeInfo.getMapValueTypeInfo()); // initialize 2 ListColumnVector for keys and values ListColumnVector keyListColumnVector = new ListColumnVector(); ListColumnVector valueListColumnVector = new ListColumnVector(); // read the keys and values keyColumnReader.readBatch(total, keyListColumnVector, keyListTypeInfo); valueColumnReader.readBatch(total, valueListColumnVector, valueListTypeInfo); // set the related attributes according to the keys and values mapColumnVector.keys = keyListColumnVector.child; mapColumnVector.values = valueListColumnVector.child; mapColumnVector.isNull = keyListColumnVector.isNull; mapColumnVector.offsets = keyListColumnVector.offsets; mapColumnVector.lengths = keyListColumnVector.lengths; mapColumnVector.childCount = keyListColumnVector.childCount; mapColumnVector.isRepeating = keyListColumnVector.isRepeating && valueListColumnVector.isRepeating; } }