private static boolean isWritableType(TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); return isWritablePrimitiveType(primitiveCategory); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; return isWritableType(mapTypeInfo.getMapKeyTypeInfo()) && isWritableType(mapTypeInfo.getMapValueTypeInfo()); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; return isWritableType(listTypeInfo.getListElementTypeInfo()); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; return structTypeInfo.getAllStructFieldTypeInfos().stream().allMatch(HiveWriteUtils::isWritableType); } return false; }
static ListTypeInfo toStructListTypeInfo(MapTypeInfo mapTypeInfo) { final StructTypeInfo structTypeInfo = new StructTypeInfo(); structTypeInfo.setAllStructFieldNames(Lists.newArrayList("keys", "values")); structTypeInfo.setAllStructFieldTypeInfos(Lists.newArrayList( mapTypeInfo.getMapKeyTypeInfo(), mapTypeInfo.getMapValueTypeInfo())); final ListTypeInfo structListTypeInfo = new ListTypeInfo(); structListTypeInfo.setListElementTypeInfo(structTypeInfo); return structListTypeInfo; }
@Override public void readBatch(int total, ColumnVector column, TypeInfo columnType) throws IOException { MapColumnVector mapColumnVector = (MapColumnVector) column; MapTypeInfo mapTypeInfo = (MapTypeInfo) columnType; ListTypeInfo keyListTypeInfo = new ListTypeInfo(); keyListTypeInfo.setListElementTypeInfo(mapTypeInfo.getMapKeyTypeInfo()); ListTypeInfo valueListTypeInfo = new ListTypeInfo(); valueListTypeInfo.setListElementTypeInfo(mapTypeInfo.getMapValueTypeInfo()); // initialize 2 ListColumnVector for keys and values ListColumnVector keyListColumnVector = new ListColumnVector(); ListColumnVector valueListColumnVector = new ListColumnVector(); // read the keys and values keyColumnReader.readBatch(total, keyListColumnVector, keyListTypeInfo); valueColumnReader.readBatch(total, valueListColumnVector, valueListTypeInfo); // set the related attributes according to the keys and values mapColumnVector.keys = keyListColumnVector.child; mapColumnVector.values = valueListColumnVector.child; mapColumnVector.isNull = keyListColumnVector.isNull; mapColumnVector.offsets = keyListColumnVector.offsets; mapColumnVector.lengths = keyListColumnVector.lengths; mapColumnVector.childCount = keyListColumnVector.childCount; mapColumnVector.isRepeating = keyListColumnVector.isRepeating && valueListColumnVector.isRepeating; } }
if (argumentAccepted.equals(argumentPassed) || TypeInfoUtils.doPrimitiveCategoriesMatch(argumentPassed, argumentAccepted)) { if (argumentPassed.equals(TypeInfoFactory.voidTypeInfo)) { if (argumentPassed.getCategory().equals(Category.LIST) && argumentAccepted.getCategory().equals(Category.LIST)) { .getListElementTypeInfo(); TypeInfo argumentAcceptedElement = ((ListTypeInfo) argumentAccepted) .getListElementTypeInfo(); return matchCost(argumentPassedElement, argumentAcceptedElement, exact); .getMapKeyTypeInfo(); TypeInfo argumentAcceptedKey = ((MapTypeInfo) argumentAccepted) .getMapKeyTypeInfo(); TypeInfo argumentPassedValue = ((MapTypeInfo) argumentPassed) .getMapValueTypeInfo(); TypeInfo argumentAcceptedValue = ((MapTypeInfo) argumentAccepted) .getMapValueTypeInfo();
public static TypeDescription convertTypeInfo(TypeInfo info) { switch (info.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo) info; switch (pinfo.getPrimitiveCategory()) { case BOOLEAN: return TypeDescription.createBoolean(); " category " + pinfo.getPrimitiveCategory()); ListTypeInfo linfo = (ListTypeInfo) info; return TypeDescription.createList (convertTypeInfo(linfo.getListElementTypeInfo())); (convertTypeInfo(minfo.getMapKeyTypeInfo()), convertTypeInfo(minfo.getMapValueTypeInfo())); for (TypeInfo child: minfo.getAllUnionObjectTypeInfos()) { result.addUnionChild(convertTypeInfo(child)); StructTypeInfo sinfo = (StructTypeInfo) info; TypeDescription result = TypeDescription.createStruct(); for(String fieldName: sinfo.getAllStructFieldNames()) { result.addField(fieldName, convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName))); info.getCategory());
switch (((ListTypeInfo) parameters[0]).getListElementTypeInfo().getCategory()) { case PRIMITIVE: pti = (PrimitiveTypeInfo) ((ListTypeInfo) parameters[0]).getListElementTypeInfo(); break; ((ListTypeInfo) parameters[0]).getListElementTypeInfo(); pti = (PrimitiveTypeInfo) lti.getListElementTypeInfo(); break; + parameters[0].getTypeName() + " was passed as parameter 1."); if(pti.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING) { throw new UDFArgumentTypeException(0, "Only array<string> or array<array<string>> is allowed, but " if(parameters[1].getCategory() != ObjectInspector.Category.LIST || ((ListTypeInfo) parameters[1]).getListElementTypeInfo().getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only arrays of strings are accepted but " + parameters[1].getTypeName() + " was passed as parameter 2."); if(((PrimitiveTypeInfo) ((ListTypeInfo)parameters[1]).getListElementTypeInfo()). getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING) { throw new UDFArgumentTypeException(1, "Only arrays of strings are accepted but " + parameters[1].getTypeName() + " was passed as parameter 2."); + parameters[2].getTypeName() + " was passed as parameter 3.");
final TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo(); return new Field(name, FieldType.nullable(MinorType.LIST.getType()), Lists.newArrayList(toField(DEFAULT_ARROW_FIELD_NAME, elementTypeInfo))); case STRUCT: final StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; final List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); final List<String> fieldNames = structTypeInfo.getAllStructFieldNames(); final List<Field> structFields = Lists.newArrayList(); final int structSize = fieldNames.size(); case UNION: final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); final List<Field> unionFields = Lists.newArrayList(); final int unionSize = unionFields.size(); case MAP: final MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; final TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo(); final TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); final StructTypeInfo mapStructTypeInfo = new StructTypeInfo(); Lists.newArrayList(keyTypeInfo, valueTypeInfo)); final ListTypeInfo mapListStructTypeInfo = new ListTypeInfo(); mapListStructTypeInfo.setListElementTypeInfo(mapStructTypeInfo);
List<TypeInfo> unionTypeInfos = ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos(); while (index < unionTypeInfos.size() && !unionTypeInfos.get(index).equals(objectTypeInfo)) { index++; GenericData.Array array = ((GenericData.Array) o); TypeInfo listTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo(); return array.stream().map((element) -> convertToORCObject(listTypeInfo, element)).collect(Collectors.toList()); TypeInfo keyInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); TypeInfo valueInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); throw new IllegalArgumentException("Error converting object of type " + o.getClass().getName() + " to ORC type " + typeInfo.getTypeName()); } else { return null;
switch (type.getCategory()) { case PRIMITIVE: switch (((PrimitiveTypeInfo) type).getPrimitiveCategory()) { case BOOLEAN: oi = new PhoenixBooleanObjectInspector(); (ListTypeInfo) type).getListElementTypeInfo(), serdeParams);
private static Type getProjectedType(TypeInfo colType, Type fieldType) { switch (colType.getCategory()) { case STRUCT: List<Type> groupFields = getProjectedGroupFields( fieldType.asGroupType(), ((StructTypeInfo) colType).getAllStructFieldNames(), ((StructTypeInfo) colType).getAllStructFieldTypeInfos() ); .named(fieldType.getName()); case LIST: TypeInfo elemType = ((ListTypeInfo) colType).getListElementTypeInfo(); if (elemType.getCategory() == ObjectInspector.Category.STRUCT) { Type subFieldType = fieldType.asGroupType().getType(0); if (!subFieldType.isPrimitive()) {
boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); if (isList) { objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); if (objectTypeInfo.getCategory() != Category.STRUCT) { throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); if (isList) { t = TypeInfoFactory.getListTypeInfo(t); if (myt.getCategory() == Category.LIST) { TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else if (myt.getCategory() == Category.MAP) { if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), ((MapTypeInfo) myt).getMapKeyTypeInfo())) { throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE .getMsg(expr)); TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else {
boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); if (isList) { objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); if (objectTypeInfo.getCategory() != Category.STRUCT) { throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); if (isList) { t = TypeInfoFactory.getListTypeInfo(t); if (myt.getCategory() == Category.LIST) { TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else if (myt.getCategory() == Category.MAP) { if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), ((MapTypeInfo) myt).getMapKeyTypeInfo())) { throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE .getMsg(expr)); TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else {
private TypeInfo extractListCompatibleType(TypeInfo hiveTypeInfo) { if (hiveTypeInfo != null && hiveTypeInfo instanceof ListTypeInfo) { return ((ListTypeInfo) hiveTypeInfo).getListElementTypeInfo(); } else { return hiveTypeInfo; //to handle map can read list of struct data (i.e. list<struct<key, value>> --> map<key, // value>) } }
/** * TODO: Handle 1) cast 2), Windowing Agg Call */ @Override /* * Handles expr like struct(key,value).key * Follows same rules as TypeCheckProcFactory::getXpathOrFuncExprNodeDesc() * which is equivalent version of parsing such an expression from AST */ public ExprNodeDesc visitFieldAccess(RexFieldAccess fieldAccess) { ExprNodeDesc parent = fieldAccess.getReferenceExpr().accept(this); String child = fieldAccess.getField().getName(); TypeInfo parentType = parent.getTypeInfo(); // Allow accessing a field of list element structs directly from a list boolean isList = (parentType.getCategory() == ObjectInspector.Category.LIST); if (isList) { parentType = ((ListTypeInfo) parentType).getListElementTypeInfo(); } TypeInfo t = ((StructTypeInfo) parentType).getStructFieldTypeInfo(child); return new ExprNodeFieldDesc(t, parent, child, isList); }
public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType); }
((ListTypeInfo) hiveType).getListElementTypeInfo() : (hiveType instanceof MapTypeInfo) ? ((MapTypeInfo) hiveType).getMapValueTypeInfo() : hiveType; String typeName = TypeInfoUtils.getBaseName(realHiveType.getTypeName());
if (lazyObject != null) { throw new RuntimeException("Expected object is null but object is not null " + lazyObject.toString() + " typeInfo " + typeInfo.toString()); "\" typeInfo " + typeInfo.toString() + " but object is null"); " and actual object " + lazyObject.toString() + " is not equal typeInfo " + typeInfo.toString()); Object primitiveObject = ((LazyPrimitive) lazyObject).getObject(); PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; switch (primitiveTypeInfo.getPrimitiveCategory()) { case BOOLEAN: throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory()); list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")" + " elementTypeInfo " + listTypeInfo.getListElementTypeInfo().toString());
@Override public boolean equals(Object other) { if (this == other) { return true; } if (!(other instanceof ListTypeInfo)) { return false; } return getListElementTypeInfo().equals( ((ListTypeInfo) other).getListElementTypeInfo()); }
public static boolean lazyCompareList(ListTypeInfo listTypeInfo, List<Object> list, List<Object> expectedList) { TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo(); final int size = list.size(); for (int i = 0; i < size; i++) { Object lazyEleObj = list.get(i); Object expectedEleObj = expectedList.get(i); if (!lazyCompare(elementTypeInfo, lazyEleObj, expectedEleObj)) { throw new RuntimeException("List element deserialized value does not match elementTypeInfo " + elementTypeInfo.toString()); } } return true; }
ListTypeInfo listTypeInfo = new ListTypeInfo(); listTypeInfo.setListElementTypeInfo(elementTypeInfo); typeInfo = listTypeInfo; } else { MapTypeInfo mapTypeInfo = new MapTypeInfo(); mapTypeInfo.setMapKeyTypeInfo(keyTypeInfo); mapTypeInfo.setMapValueTypeInfo(elementTypeInfo); typeInfo = mapTypeInfo; final String typeName = typeInfo.getTypeName();