public static boolean lazyCompareMap(MapTypeInfo mapTypeInfo, Map<Object, Object> map, Map<Object, Object> expectedMap) { TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo(); TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); if (map.size() != expectedMap.size()) { throw new RuntimeException("Map key/value deserialized map.size() " + map.size() + " map " + map.toString() + " expectedMap.size() " + expectedMap.size() + " expectedMap " + expectedMap.toString() + " does not match keyTypeInfo " + keyTypeInfo.toString() + " valueTypeInfo " + valueTypeInfo.toString()); } return true; }
@Override public boolean equals(Object other) { if (this == other) { return true; } if (!(other instanceof MapTypeInfo)) { return false; } MapTypeInfo o = (MapTypeInfo) other; return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType); } }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType); } }
private Schema createAvroMap(TypeInfo typeInfo) { TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); if (((PrimitiveTypeInfo) keyTypeInfo).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING) { throw new UnsupportedOperationException("Key of Map can only be a String"); } TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); Schema valueSchema = createAvroSchema(valueTypeInfo); return Schema.createMap(valueSchema); }
private static int hashOfMap(MapTypeInfo type, Block singleMapBlock) { TypeInfo keyTypeInfo = type.getMapKeyTypeInfo(); TypeInfo valueTypeInfo = type.getMapValueTypeInfo(); int result = 0; for (int i = 0; i < singleMapBlock.getPositionCount(); i += 2) { result += hash(keyTypeInfo, singleMapBlock, i) ^ hash(valueTypeInfo, singleMapBlock, i + 1); } return result; }
OrcMapObjectInspector(MapTypeInfo info) { key = createObjectInspector(info.getMapKeyTypeInfo()); value = createObjectInspector(info.getMapValueTypeInfo()); }
static ListTypeInfo toStructListTypeInfo(MapTypeInfo mapTypeInfo) { final StructTypeInfo structTypeInfo = new StructTypeInfo(); structTypeInfo.setAllStructFieldNames(Lists.newArrayList("keys", "values")); structTypeInfo.setAllStructFieldTypeInfos(Lists.newArrayList( mapTypeInfo.getMapKeyTypeInfo(), mapTypeInfo.getMapValueTypeInfo())); final ListTypeInfo structListTypeInfo = new ListTypeInfo(); structListTypeInfo.setListElementTypeInfo(structTypeInfo); return structListTypeInfo; }
public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); this.toType = requireNonNull(toHiveType, "toHiveType is null").getType(typeManager); HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); this.keyCoercer = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType); this.valueCoercer = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType); }
public static RelDataType convert(MapTypeInfo mapType, RelDataTypeFactory dtFactory) throws CalciteSemanticException { RelDataType keyType = convert(mapType.getMapKeyTypeInfo(), dtFactory); RelDataType valueType = convert(mapType.getMapValueTypeInfo(), dtFactory); return dtFactory.createMapType(keyType, valueType); }
public static boolean isSupportedType(TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: return getPrimitiveType((PrimitiveTypeInfo) typeInfo) != null; case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; return isSupportedType(mapTypeInfo.getMapKeyTypeInfo()) && isSupportedType(mapTypeInfo.getMapValueTypeInfo()); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; return isSupportedType(listTypeInfo.getListElementTypeInfo()); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; return structTypeInfo.getAllStructFieldTypeInfos().stream() .allMatch(HiveType::isSupportedType); } return false; }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return mapType(Repetition.OPTIONAL, name, "map", keyType, valueType); }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo, final Repetition repetition) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return ConversionPatterns.mapType(repetition, name, keyType, valueType); }
public static RelDataType convert(MapTypeInfo mapType, RelDataTypeFactory dtFactory) throws CalciteSemanticException { RelDataType keyType = convert(mapType.getMapKeyTypeInfo(), dtFactory); RelDataType valueType = convert(mapType.getMapValueTypeInfo(), dtFactory); return dtFactory.createMapType(keyType, valueType); }
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo, final Repetition repetition) { final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED); final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo()); return mapType(repetition, name, "map", keyType, valueType); }
private static boolean isWritableType(TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); return isWritablePrimitiveType(primitiveCategory); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; return isWritableType(mapTypeInfo.getMapKeyTypeInfo()) && isWritableType(mapTypeInfo.getMapValueTypeInfo()); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; return isWritableType(listTypeInfo.getListElementTypeInfo()); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; return structTypeInfo.getAllStructFieldTypeInfos().stream().allMatch(HiveWriteUtils::isWritableType); } return false; }
public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType, BridgingRecordCursor bridgingRecordCursor) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); this.bridgingRecordCursor = requireNonNull(bridgingRecordCursor, "bridgingRecordCursor is null"); HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); this.fromKeyValueTypes = fromHiveType.getType(typeManager).getTypeParameters(); this.toType = toHiveType.getType(typeManager); this.toKeyValueTypes = toType.getTypeParameters(); this.coercers = new Coercer[2]; coercers[0] = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType, bridgingRecordCursor); coercers[1] = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType, bridgingRecordCursor); this.pageBuilder = coercers[0] == null && coercers[1] == null ? null : new PageBuilder(ImmutableList.of(toType)); }
private static void validateAvroType(TypeInfo type, String columnName) { if (type.getCategory() == Category.MAP) { TypeInfo keyType = mapTypeInfo(type).getMapKeyTypeInfo(); if ((keyType.getCategory() != Category.PRIMITIVE) || (primitiveTypeInfo(keyType).getPrimitiveCategory() != PrimitiveCategory.STRING)) { throw new PrestoException(NOT_SUPPORTED, format("Column %s has a non-varchar map key, which is not supported by Avro", columnName)); } } else if (type.getCategory() == Category.PRIMITIVE) { PrimitiveCategory primitive = primitiveTypeInfo(type).getPrimitiveCategory(); if (primitive == PrimitiveCategory.BYTE) { throw new PrestoException(NOT_SUPPORTED, format("Column %s is tinyint, which is not supported by Avro. Use integer instead.", columnName)); } if (primitive == PrimitiveCategory.SHORT) { throw new PrestoException(NOT_SUPPORTED, format("Column %s is smallint, which is not supported by Avro. Use integer instead.", columnName)); } } }
public KeyValueConverter(GroupType keyValueType, HiveGroupConverter parent, TypeInfo hiveTypeInfo) { setMetadata(parent.getMetadata()); this.parent = parent; this.keyConverter = getConverterFromDescription( keyValueType.getType(0), 0, this, hiveTypeInfo == null ? null : ((MapTypeInfo) hiveTypeInfo).getMapKeyTypeInfo()); this.valueConverter = getConverterFromDescription( keyValueType.getType(1), 1, this, hiveTypeInfo == null ? null : ((MapTypeInfo) hiveTypeInfo).getMapValueTypeInfo()); }
private boolean canCoerceForMap(HiveType fromHiveType, HiveType toHiveType) { if (!fromHiveType.getCategory().equals(Category.MAP) || !toHiveType.getCategory().equals(Category.MAP)) { return false; } HiveType fromKeyType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); return (fromKeyType.equals(toKeyType) || canCoerce(fromKeyType, toKeyType)) && (fromValueType.equals(toValueType) || canCoerce(fromValueType, toValueType)); }