if (oi == null) { LOG.debug("Got asked for OI for {}, [{}]", typeInfo.getCategory(), typeInfo.getTypeName()); switch (typeInfo.getCategory()) { case PRIMITIVE: oi = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List<String> fieldNames = structTypeInfo.getAllStructFieldNames(); List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size()); case LIST: ObjectInspector elementObjectInspector = getStandardObjectInspectorFromTypeInfo( ((ListTypeInfo) typeInfo).getListElementTypeInfo()); oi = ObjectInspectorFactory.getStandardListObjectInspector(elementObjectInspector); break; case MAP: ObjectInspector keyObjectInspector = getStandardObjectInspectorFromTypeInfo( ((MapTypeInfo) typeInfo).getMapKeyTypeInfo()); ObjectInspector valueObjectInspector = getStandardObjectInspectorFromTypeInfo( ((MapTypeInfo) typeInfo).getMapValueTypeInfo()); oi = ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector); break;
private static HCatSchema constructHCatSchema(StructTypeInfo stypeInfo) throws HCatException { CollectionBuilder builder = getStructSchemaBuilder(); for (String fieldName : stypeInfo.getAllStructFieldNames()) { builder.addField(getHCatFieldSchema(fieldName, stypeInfo.getStructFieldTypeInfo(fieldName), null)); } return builder.build(); }
static ListTypeInfo toStructListTypeInfo(MapTypeInfo mapTypeInfo) { final StructTypeInfo structTypeInfo = new StructTypeInfo(); structTypeInfo.setAllStructFieldNames(Lists.newArrayList("keys", "values")); structTypeInfo.setAllStructFieldTypeInfos(Lists.newArrayList( mapTypeInfo.getMapKeyTypeInfo(), mapTypeInfo.getMapValueTypeInfo())); final ListTypeInfo structListTypeInfo = new ListTypeInfo(); structListTypeInfo.setListElementTypeInfo(structTypeInfo); return structListTypeInfo; }
@SuppressWarnings({"unchecked"}) private static ArrayList<? extends Object>[] getTypeMap( StructObjectInspector oi) { StructTypeInfo t = (StructTypeInfo) TypeInfoUtils .getTypeInfoFromObjectInspector(oi); ArrayList<String> fnames = t.getAllStructFieldNames(); ArrayList<TypeInfo> fields = t.getAllStructFieldTypeInfos(); return new ArrayList<?>[] {fnames, fields}; }
PrunedStructTypeInfo(StructTypeInfo typeInfo) { this.typeInfo = typeInfo; this.children = new HashMap<>(); this.selected = new boolean[typeInfo.getAllStructFieldTypeInfos().size()]; for (int i = 0; i < typeInfo.getAllStructFieldTypeInfos().size(); ++i) { TypeInfo ti = typeInfo.getAllStructFieldTypeInfos().get(i); if (ti.getCategory() == Category.STRUCT) { this.children.put(typeInfo.getAllStructFieldNames().get(i).toLowerCase(), new PrunedStructTypeInfo((StructTypeInfo) ti)); } } }
public static TypeDescription convertTypeInfo(TypeInfo info) { switch (info.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo) info; switch (pinfo.getPrimitiveCategory()) { case BOOLEAN: return TypeDescription.createBoolean(); " category " + pinfo.getPrimitiveCategory()); ListTypeInfo linfo = (ListTypeInfo) info; return TypeDescription.createList (convertTypeInfo(linfo.getListElementTypeInfo())); (convertTypeInfo(minfo.getMapKeyTypeInfo()), convertTypeInfo(minfo.getMapValueTypeInfo())); for (TypeInfo child: minfo.getAllUnionObjectTypeInfos()) { result.addUnionChild(convertTypeInfo(child)); StructTypeInfo sinfo = (StructTypeInfo) info; TypeDescription result = TypeDescription.createStruct(); for(String fieldName: sinfo.getAllStructFieldNames()) { result.addField(fieldName, convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName))); info.getCategory());
final TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo(); return new Field(name, FieldType.nullable(MinorType.LIST.getType()), Lists.newArrayList(toField(DEFAULT_ARROW_FIELD_NAME, elementTypeInfo))); case STRUCT: final StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; final List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); final List<String> fieldNames = structTypeInfo.getAllStructFieldNames(); final List<Field> structFields = Lists.newArrayList(); final int structSize = fieldNames.size(); case UNION: final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); final List<Field> unionFields = Lists.newArrayList(); final int unionSize = unionFields.size(); case MAP: final MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; final TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo(); final TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); final StructTypeInfo mapStructTypeInfo = new StructTypeInfo(); mapStructTypeInfo.setAllStructFieldNames(Lists.newArrayList("keys", "values")); mapStructTypeInfo.setAllStructFieldTypeInfos( Lists.newArrayList(keyTypeInfo, valueTypeInfo)); final ListTypeInfo mapListStructTypeInfo = new ListTypeInfo(); mapListStructTypeInfo.setListElementTypeInfo(mapStructTypeInfo);
for (ExprNodeDesc d : desc.getChildren()) { TypeInfo typeInfo = d.getTypeInfo(); if (typeInfo.getCategory() != Category.STRUCT) { return false; .getAllStructFieldTypeInfos(); ArrayList<String> fieldNames = structTypeInfo.getAllStructFieldNames(); final int fieldCount = fieldTypeInfos.size(); for (int f = 0; f < fieldCount; f++) { TypeInfo fieldTypeInfo = fieldTypeInfos.get(f); Category category = fieldTypeInfo.getCategory(); if (category != Category.PRIMITIVE) { setExpressionIssue(expressionTitle, "Cannot vectorize struct field " + fieldNames.get(f) + " of type " + fieldTypeInfo.getTypeName()); return false; InConstantType inConstantType = VectorizationContext .getInConstantTypeFromPrimitiveCategory(fieldPrimitiveTypeInfo .getPrimitiveCategory());
boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); if (isList) { objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); if (objectTypeInfo.getCategory() != Category.STRUCT) { throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); if (isList) { t = TypeInfoFactory.getListTypeInfo(t); if (myt.getCategory() == Category.LIST) { TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else if (myt.getCategory() == Category.MAP) { if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), ((MapTypeInfo) myt).getMapKeyTypeInfo())) { throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE .getMsg(expr)); TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else {
boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); if (isList) { objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); if (objectTypeInfo.getCategory() != Category.STRUCT) { throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); if (isList) { t = TypeInfoFactory.getListTypeInfo(t); if (myt.getCategory() == Category.LIST) { TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); } else { children.size()); for (ExprNodeDesc child : children) { if (TypeInfoFactory.getPrimitiveTypeInfo("void").equals(child.getTypeInfo())) { child.setTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("boolean")); children.size()); for (ExprNodeDesc child : children) { if (TypeInfoFactory.getPrimitiveTypeInfo("void").equals(child.getTypeInfo())) { child.setTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("boolean"));
if (oi == null) { LOG.debug("Got asked for OI for {} [{} ]", typeInfo.getCategory(), typeInfo.getTypeName()); switch (typeInfo.getCategory()) { case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List<String> fieldNames = structTypeInfo.getAllStructFieldNames(); List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size()); for (int i = 0; i < fieldTypeInfos.size(); i++) { throw new SerDeException("TypeInfo [" + typeInfo.getTypeName() + "] was not of struct type - HCatRecord expected struct type, got [" + typeInfo.getCategory().toString() + "]");
private static Type getProjectedType(TypeInfo colType, Type fieldType) { switch (colType.getCategory()) { case STRUCT: List<Type> groupFields = getProjectedGroupFields( fieldType.asGroupType(), ((StructTypeInfo) colType).getAllStructFieldNames(), ((StructTypeInfo) colType).getAllStructFieldTypeInfos() ); .named(fieldType.getName()); case LIST: TypeInfo elemType = ((ListTypeInfo) colType).getListElementTypeInfo(); if (elemType.getCategory() == ObjectInspector.Category.STRUCT) { Type subFieldType = fieldType.asGroupType().getType(0); if (!subFieldType.isPrimitive()) {
switch (fieldTypeInfo.getCategory()) { case PRIMITIVE: PrimitiveObjectInspector.PrimitiveCategory primitiveCategory = PrimitiveObjectInspector.PrimitiveCategory.UNKNOWN; if (fieldTypeInfo instanceof PrimitiveTypeInfo) { primitiveCategory = ((PrimitiveTypeInfo) fieldTypeInfo).getPrimitiveCategory(); StructTypeInfo typeInfo = (StructTypeInfo) schema.getStructFieldTypeInfo(fieldName); for (RecordField nestedRecordField : recordFields) { String fName = nestedRecordField.getFieldName(); String normalizedFieldName = fName.toLowerCase(); structList.add(extractCurrentField(nestedRecord, nestedRecordField, typeInfo.getStructFieldTypeInfo(normalizedFieldName)));
/** * TODO: Handle 1) cast 2), Windowing Agg Call */ @Override /* * Handles expr like struct(key,value).key * Follows same rules as TypeCheckProcFactory::getXpathOrFuncExprNodeDesc() * which is equivalent version of parsing such an expression from AST */ public ExprNodeDesc visitFieldAccess(RexFieldAccess fieldAccess) { ExprNodeDesc parent = fieldAccess.getReferenceExpr().accept(this); String child = fieldAccess.getField().getName(); TypeInfo parentType = parent.getTypeInfo(); // Allow accessing a field of list element structs directly from a list boolean isList = (parentType.getCategory() == ObjectInspector.Category.LIST); if (isList) { parentType = ((ListTypeInfo) parentType).getListElementTypeInfo(); } TypeInfo t = ((StructTypeInfo) parentType).getStructFieldTypeInfo(child); return new ExprNodeFieldDesc(t, parent, child, isList); }
ArrayList<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); final int fieldCount = fieldTypeInfos.size(); ColumnVector.Type[] fieldVectorColumnTypes = new ColumnVector.Type[fieldCount]; TypeInfo fieldTypeInfo = fieldTypeInfos.get(f); if (fieldTypeInfo.getCategory() != Category.PRIMITIVE) { return null; ((PrimitiveTypeInfo) fieldTypeInfo).getPrimitiveCategory(); InConstantType inConstantType = getInConstantTypeFromPrimitiveCategory(fieldPrimitiveCategory); fieldInConstantTypes[f] = inConstantType;
@Override /** * Return string representation of constant expression * Beside ExplainPlan task Default constraint also make use it to deserialize constant expression * to store it in metastore, which is later reparsed to generate appropriate constant expression * Therefore it is necessary for this method to qualify the intervals with appropriate qualifiers */ public String getExprString() { if (typeInfo.getCategory() == Category.PRIMITIVE) { return getFormatted(typeInfo, value); } else if (typeInfo.getCategory() == Category.STRUCT) { StringBuilder sb = new StringBuilder(); sb.append("const struct("); List<?> items = (List<?>) getWritableObjectInspector().getWritableConstantValue(); List<TypeInfo> structTypes = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); for (int i = 0; i < structTypes.size(); i++) { final Object o = (i < items.size()) ? items.get(i) : null; sb.append(getFormatted(structTypes.get(i), o)).append(","); } sb.setCharAt(sb.length() - 1, ')'); return sb.toString(); } else { // unknown type return toString(); } }
private Schema createAvroRecord(TypeInfo typeInfo) { List<Schema.Field> childFields = new ArrayList<Schema.Field>(); final List<String> allStructFieldNames = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); final List<TypeInfo> allStructFieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + "name and column type differs. names = " + allStructFieldNames + ", types = " + allStructFieldTypeInfos); } for (int i = 0; i < allStructFieldNames.size(); ++i) { final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), childTypeInfo, childTypeInfo.toString()); final List<Schema.Field> grandChildFields = getFields(grandChildSchemaField); childFields.addAll(grandChildFields); } Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), null, false); ++recordCounter; recordSchema.setFields(childFields); return recordSchema; }
public static List<HiveType> extractStructFieldTypes(HiveType hiveType) { return ((StructTypeInfo) hiveType.getTypeInfo()).getAllStructFieldTypeInfos().stream() .map(typeInfo -> HiveType.valueOf(typeInfo.getTypeName())) .collect(toImmutableList()); }
ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc; StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo(); ArrayList<TypeInfo> structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos(); ArrayList<TypeInfo> newStructFieldInfos = new ArrayList<>(); newValues.add(newValue); StructTypeInfo sti = new StructTypeInfo(); sti.setAllStructFieldTypeInfos(newStructFieldInfos); sti.setAllStructFieldNames(structTypeInfo.getAllStructFieldNames()); return new ExprNodeConstantDesc(sti, newValues);
void markSelected(String fieldName) { for (int i = 0; i < typeInfo.getAllStructFieldNames().size(); ++i) { if (typeInfo.getAllStructFieldNames().get(i).equalsIgnoreCase(fieldName)) { selected[i] = true; break; } } }