private static boolean isWritableType(TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); return isWritablePrimitiveType(primitiveCategory); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; return isWritableType(mapTypeInfo.getMapKeyTypeInfo()) && isWritableType(mapTypeInfo.getMapValueTypeInfo()); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; return isWritableType(listTypeInfo.getListElementTypeInfo()); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; return structTypeInfo.getAllStructFieldTypeInfos().stream().allMatch(HiveWriteUtils::isWritableType); } return false; }
private static void checkArgumentTypes(TypeInfo[] parameters) throws UDFArgumentTypeException { if (parameters.length != 2) { throw new UDFArgumentTypeException(parameters.length - 1, "Exactly two arguments are expected."); } if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); } if (parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + parameters[1].getTypeName() + " is passed."); } if (!acceptedPrimitiveCategory(((PrimitiveTypeInfo) parameters[0]).getPrimitiveCategory())) { throw new UDFArgumentTypeException(0, "Only numeric type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); } if (!acceptedPrimitiveCategory(((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory())) { throw new UDFArgumentTypeException(1, "Only numeric type arguments are accepted but " + parameters[1].getTypeName() + " is passed."); } }
@Override public boolean equals(Object other) { if (this == other) { return true; } if (!(other instanceof MapTypeInfo)) { return false; } MapTypeInfo o = (MapTypeInfo) other; return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); }
@Override public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException { if (parameters.length != 1) { throw new IllegalArgumentException("Function only takes 1 parameter"); } else if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE && parameters[0].getCategory() != ObjectInspector.Category.STRUCT) { throw new UDFArgumentTypeException(1, "Only primitive/struct rows are accepted but " + parameters[0].getTypeName() + " was passed."); } return new CountApproximateDistinctEvaluator(); } }
static String getScratchName(TypeInfo typeInfo) throws HiveException { // For now, leave DECIMAL precision/scale in the name so DecimalColumnVector scratch columns // don't need their precision/scale adjusted... if (typeInfo.getCategory() == Category.PRIMITIVE && ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.DECIMAL) { return typeInfo.getTypeName(); } // And, for Complex Types, also leave the children types in place... if (typeInfo.getCategory() != Category.PRIMITIVE) { return typeInfo.getTypeName(); } Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo); return columnVectorType.name().toLowerCase(); }
public static TypeDescription convertTypeInfo(TypeInfo info) { switch (info.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo) info; switch (pinfo.getPrimitiveCategory()) { case BOOLEAN: return TypeDescription.createBoolean(); " category " + pinfo.getPrimitiveCategory()); ListTypeInfo linfo = (ListTypeInfo) info; return TypeDescription.createList (convertTypeInfo(linfo.getListElementTypeInfo())); (convertTypeInfo(minfo.getMapKeyTypeInfo()), convertTypeInfo(minfo.getMapValueTypeInfo())); for (TypeInfo child: minfo.getAllUnionObjectTypeInfos()) { result.addUnionChild(convertTypeInfo(child)); StructTypeInfo sinfo = (StructTypeInfo) info; TypeDescription result = TypeDescription.createStruct(); for(String fieldName: sinfo.getAllStructFieldNames()) { result.addField(fieldName, convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName))); info.getCategory());
private static Field toField(String name, TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: final PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; switch (primitiveTypeInfo.getPrimitiveCategory()) { case BOOLEAN: return Field.nullable(name, MinorType.BIT.getType()); case UNION: final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); final List<Field> unionFields = Lists.newArrayList(); final int unionSize = unionFields.size(); case MAP: final MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; final TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo(); final TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); final StructTypeInfo mapStructTypeInfo = new StructTypeInfo(); Lists.newArrayList(keyTypeInfo, valueTypeInfo)); final ListTypeInfo mapListStructTypeInfo = new ListTypeInfo(); mapListStructTypeInfo.setListElementTypeInfo(mapStructTypeInfo);
private static HCatFieldSchema getHCatFieldSchema(String fieldName, TypeInfo fieldTypeInfo, String comment) throws HCatException { Category typeCategory = fieldTypeInfo.getCategory(); HCatFieldSchema hCatFieldSchema; if (Category.PRIMITIVE == typeCategory) { hCatFieldSchema = new HCatFieldSchema(fieldName, (PrimitiveTypeInfo)fieldTypeInfo, comment); } else if (Category.STRUCT == typeCategory) { HCatSchema subSchema = constructHCatSchema((StructTypeInfo) fieldTypeInfo); hCatFieldSchema = new HCatFieldSchema(fieldName, HCatFieldSchema.Type.STRUCT, subSchema, comment); } else if (Category.LIST == typeCategory) { HCatSchema subSchema = getHCatSchema(((ListTypeInfo) fieldTypeInfo).getListElementTypeInfo()); hCatFieldSchema = new HCatFieldSchema(fieldName, HCatFieldSchema.Type.ARRAY, subSchema, comment); } else if (Category.MAP == typeCategory) { HCatSchema subSchema = getHCatSchema(((MapTypeInfo) fieldTypeInfo).getMapValueTypeInfo()); hCatFieldSchema = HCatFieldSchema.createMapTypeFieldSchema(fieldName, (PrimitiveTypeInfo)((MapTypeInfo)fieldTypeInfo).getMapKeyTypeInfo(), subSchema, comment); } else { throw new TypeNotPresentException(fieldTypeInfo.getTypeName(), null); } return hCatFieldSchema; }
public boolean isCategory(PrimitiveCategory category) { return columnType.getCategory() == ObjectInspector.Category.PRIMITIVE && ((PrimitiveTypeInfo)columnType).getPrimitiveCategory() == category; }
public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); this.toType = requireNonNull(toHiveType, "toHiveType is null").getType(typeManager); HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); this.keyCoercer = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType); this.valueCoercer = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType); }
public void init(List<String> typeNames) throws HiveException { final int size = typeNames.size(); categories = new Category[size]; primitiveCategories = new PrimitiveCategory[size]; outputColumnNums = new int[size]; TypeInfo typeInfo; for (int i = 0; i < size; i++) { typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeNames.get(i)); categories[i] = typeInfo.getCategory(); if (categories[i] == Category.PRIMITIVE) { primitiveCategories[i] = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); } outputColumnNums[i] = i; } }
PrunedStructTypeInfo(StructTypeInfo typeInfo) { this.typeInfo = typeInfo; this.children = new HashMap<>(); this.selected = new boolean[typeInfo.getAllStructFieldTypeInfos().size()]; for (int i = 0; i < typeInfo.getAllStructFieldTypeInfos().size(); ++i) { TypeInfo ti = typeInfo.getAllStructFieldTypeInfos().get(i); if (ti.getCategory() == Category.STRUCT) { this.children.put(typeInfo.getAllStructFieldNames().get(i).toLowerCase(), new PrunedStructTypeInfo((StructTypeInfo) ti)); } } }
private static void validateAvroType(TypeInfo type, String columnName) { if (type.getCategory() == Category.MAP) { TypeInfo keyType = mapTypeInfo(type).getMapKeyTypeInfo(); if ((keyType.getCategory() != Category.PRIMITIVE) || (primitiveTypeInfo(keyType).getPrimitiveCategory() != PrimitiveCategory.STRING)) { throw new PrestoException(NOT_SUPPORTED, format("Column %s has a non-varchar map key, which is not supported by Avro", columnName)); } } else if (type.getCategory() == Category.PRIMITIVE) { PrimitiveCategory primitive = primitiveTypeInfo(type).getPrimitiveCategory(); if (primitive == PrimitiveCategory.BYTE) { throw new PrestoException(NOT_SUPPORTED, format("Column %s is tinyint, which is not supported by Avro. Use integer instead.", columnName)); } if (primitive == PrimitiveCategory.SHORT) { throw new PrestoException(NOT_SUPPORTED, format("Column %s is smallint, which is not supported by Avro. Use integer instead.", columnName)); } } }
private TypeInfo getFieldTypeIgnoreCase(TypeInfo hiveTypeInfo, String fieldName, int fieldIndex) { if (hiveTypeInfo == null) { return null; } else if (hiveTypeInfo.getCategory().equals(ObjectInspector.Category.STRUCT)) { return getStructFieldTypeInfo(fieldName, fieldIndex); } else if (hiveTypeInfo.getCategory().equals(ObjectInspector.Category.MAP)) { //This cover the case where hive table may have map<key, value> but the data file is // of type array<struct<value1, value2>> //Using index in place of type name. if (fieldIndex == 0) { return ((MapTypeInfo) hiveTypeInfo).getMapKeyTypeInfo(); } else if (fieldIndex == 1) { return ((MapTypeInfo) hiveTypeInfo).getMapValueTypeInfo(); } else {//Other fields are skipped for this case return null; } } throw new RuntimeException("Unknown hive type info " + hiveTypeInfo + " when searching for field " + fieldName); }
/** * TODO: Handle 1) cast 2), Windowing Agg Call */ @Override /* * Handles expr like struct(key,value).key * Follows same rules as TypeCheckProcFactory::getXpathOrFuncExprNodeDesc() * which is equivalent version of parsing such an expression from AST */ public ExprNodeDesc visitFieldAccess(RexFieldAccess fieldAccess) { ExprNodeDesc parent = fieldAccess.getReferenceExpr().accept(this); String child = fieldAccess.getField().getName(); TypeInfo parentType = parent.getTypeInfo(); // Allow accessing a field of list element structs directly from a list boolean isList = (parentType.getCategory() == ObjectInspector.Category.LIST); if (isList) { parentType = ((ListTypeInfo) parentType).getListElementTypeInfo(); } TypeInfo t = ((StructTypeInfo) parentType).getStructFieldTypeInfo(child); return new ExprNodeFieldDesc(t, parent, child, isList); }
public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType); }
public static List<String> getColumnTypes(Properties props) { List<String> names = new ArrayList<String>(); String colNames = props.getProperty(serdeConstants.LIST_COLUMN_TYPES); ArrayList<TypeInfo> cols = TypeInfoUtils.getTypeInfosFromTypeString(colNames); for (TypeInfo col : cols) { names.add(col.getTypeName()); } return names; }
public static boolean validateDataType(String type, VectorExpressionDescriptor.Mode mode, boolean allowComplex, boolean allowVoidProjection) { type = type.toLowerCase(); boolean result = supportedDataTypesPattern.matcher(type).matches(); if (result && !allowVoidProjection && mode == VectorExpressionDescriptor.Mode.PROJECTION && type.equals("void")) { return false; } if (!result) { TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(type); if (typeInfo.getCategory() != Category.PRIMITIVE) { if (allowComplex) { return true; } } } return result; }
public static boolean lazyCompareMap(MapTypeInfo mapTypeInfo, Map<Object, Object> map, Map<Object, Object> expectedMap) { TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo(); TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); if (map.size() != expectedMap.size()) { throw new RuntimeException("Map key/value deserialized map.size() " + map.size() + " map " + map.toString() + " expectedMap.size() " + expectedMap.size() + " expectedMap " + expectedMap.toString() + " does not match keyTypeInfo " + keyTypeInfo.toString() + " valueTypeInfo " + valueTypeInfo.toString()); } return true; }
public static List<HiveType> extractStructFieldTypes(HiveType hiveType) { return ((StructTypeInfo) hiveType.getTypeInfo()).getAllStructFieldTypeInfos().stream() .map(typeInfo -> HiveType.valueOf(typeInfo.getTypeName())) .collect(toImmutableList()); }