private Field createField(TypeInfo typeInfo) { final Field field = new Field(); final Category category = typeInfo.getCategory(); field.category = category; field.typeInfo = typeInfo; field.primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); break; case LIST: field.children = new Field[1]; field.children[0] = createField(((ListTypeInfo) typeInfo).getListElementTypeInfo()); break; case MAP: field.children = new Field[2]; field.children[0] = createField(((MapTypeInfo) typeInfo).getMapKeyTypeInfo()); field.children[1] = createField(((MapTypeInfo) typeInfo).getMapValueTypeInfo()); break; case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); field.count = fieldTypeInfos.size(); field.children = createFields(fieldTypeInfos.toArray(new TypeInfo[fieldTypeInfos.size()])); case UNION: UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); field.count = 2; field.children = createFields(objectTypeInfos.toArray(new TypeInfo[objectTypeInfos.size()]));
public static TypeInfo getUnionTypeInfo(List<TypeInfo> typeInfos) { TypeInfo result = cachedUnionTypeInfo.get(typeInfos); if (result == null) { result = new UnionTypeInfo(typeInfos); TypeInfo prev = cachedUnionTypeInfo.putIfAbsent(typeInfos, result); if (prev != null) { result = prev; } } return result; }
private static Field toField(String name, TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: final PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; switch (primitiveTypeInfo.getPrimitiveCategory()) { case BOOLEAN: return Field.nullable(name, MinorType.BIT.getType()); case UNION: final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); final List<Field> unionFields = Lists.newArrayList(); final int unionSize = unionFields.size(); case MAP: final MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; final TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo(); final TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); final StructTypeInfo mapStructTypeInfo = new StructTypeInfo(); Lists.newArrayList(keyTypeInfo, valueTypeInfo)); final ListTypeInfo mapListStructTypeInfo = new ListTypeInfo(); mapListStructTypeInfo.setListElementTypeInfo(mapStructTypeInfo);
public static TypeDescription convertTypeInfo(TypeInfo info) { switch (info.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo) info; switch (pinfo.getPrimitiveCategory()) { case BOOLEAN: return TypeDescription.createBoolean(); " category " + pinfo.getPrimitiveCategory()); ListTypeInfo linfo = (ListTypeInfo) info; return TypeDescription.createList (convertTypeInfo(linfo.getListElementTypeInfo())); (convertTypeInfo(minfo.getMapKeyTypeInfo()), convertTypeInfo(minfo.getMapValueTypeInfo())); for (TypeInfo child: minfo.getAllUnionObjectTypeInfos()) { result.addUnionChild(convertTypeInfo(child)); StructTypeInfo sinfo = (StructTypeInfo) info; TypeDescription result = TypeDescription.createStruct(); for(String fieldName: sinfo.getAllStructFieldNames()) { result.addField(fieldName, convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName))); info.getCategory());
@VisibleForTesting static Schema convert(LinkedList<String> path, String name, TypeInfo type, Collection<String[]> required) { switch (type.getCategory()) { case PRIMITIVE: if (type.getClass() == charClass || type.getClass() == varcharClass) { String typeInfoName = type.getTypeName(); Preconditions.checkArgument(TYPEINFO_TO_TYPE.containsKey(typeInfoName), "Cannot convert unsupported type: %s", typeInfoName); ((ListTypeInfo) type).getListElementTypeInfo(), required))); "string".equals(mapType.getMapKeyTypeInfo().toString()), "Non-String map key type: %s", mapType.getMapKeyTypeInfo()); mapType.getMapValueTypeInfo(), required))); .getAllUnionObjectTypeInfos();
List<TypeInfo> unionTypeInfos = ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos(); while (index < unionTypeInfos.size() && !unionTypeInfos.get(index).equals(objectTypeInfo)) { index++; GenericData.Array array = ((GenericData.Array) o); TypeInfo listTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo(); return array.stream().map((element) -> convertToORCObject(listTypeInfo, element)).collect(Collectors.toList()); TypeInfo keyInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); TypeInfo valueInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); throw new IllegalArgumentException("Error converting object of type " + o.getClass().getName() + " to ORC type " + typeInfo.getTypeName()); } else { return null;
@Override public boolean equals(Object other) { if (this == other) { return true; } if (!(other instanceof UnionTypeInfo)) { return false; } UnionTypeInfo o = (UnionTypeInfo) other; // Compare the field types return o.getAllUnionObjectTypeInfos().equals(getAllUnionObjectTypeInfos()); }
public static TypeDescription convertTypeInfo(TypeInfo info) { switch (info.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo) info; switch (pinfo.getPrimitiveCategory()) { case BOOLEAN: return TypeDescription.createBoolean(); " category " + pinfo.getPrimitiveCategory()); ListTypeInfo linfo = (ListTypeInfo) info; return TypeDescription.createList (convertTypeInfo(linfo.getListElementTypeInfo())); (convertTypeInfo(minfo.getMapKeyTypeInfo()), convertTypeInfo(minfo.getMapValueTypeInfo())); for (TypeInfo child: minfo.getAllUnionObjectTypeInfos()) { result.addUnionChild(convertTypeInfo(child)); StructTypeInfo sinfo = (StructTypeInfo) info; TypeDescription result = TypeDescription.createStruct(); for(String fieldName: sinfo.getAllStructFieldNames()) { result.addField(fieldName, convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName))); info.getCategory());
@VisibleForTesting static Schema convert(LinkedList<String> path, String name, TypeInfo type, Collection<String[]> required) { switch (type.getCategory()) { case PRIMITIVE: if (type.getClass() == charClass || type.getClass() == varcharClass) { String typeInfoName = type.getTypeName(); Preconditions.checkArgument(TYPEINFO_TO_TYPE.containsKey(typeInfoName), "Cannot convert unsupported type: %s", typeInfoName); ((ListTypeInfo) type).getListElementTypeInfo(), required))); "string".equals(mapType.getMapKeyTypeInfo().toString()), "Non-String map key type: %s", mapType.getMapKeyTypeInfo()); mapType.getMapValueTypeInfo(), required))); .getAllUnionObjectTypeInfos();
List<TypeInfo> unionTypeInfos = ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos(); while (index < unionTypeInfos.size() && !unionTypeInfos.get(index).equals(objectTypeInfo)) { index++; if(TypeInfoFactory.binaryTypeInfo.equals(typeInfo)) { byte[] dest = new byte[objArray.length]; for(int i=0;i<objArray.length;i++) { } else { TypeInfo listTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo(); return Arrays.stream(objArray) .map(o1 -> convertToORCObject(listTypeInfo, o1, hiveFieldNames)) Map map = new HashMap(); MapTypeInfo mapTypeInfo = ((MapTypeInfo) typeInfo); TypeInfo keyInfo = mapTypeInfo.getMapKeyTypeInfo(); TypeInfo valueInfo = mapTypeInfo.getMapValueTypeInfo(); return map; throw new IllegalArgumentException("Error converting object of type " + o.getClass().getName() + " to ORC type " + typeInfo.getTypeName()); } else { return null;
OrcUnionObjectInspector(UnionTypeInfo info) { List<TypeInfo> unionChildren = info.getAllUnionObjectTypeInfos(); this.children = new ArrayList<ObjectInspector>(unionChildren.size()); for(TypeInfo child: info.getAllUnionObjectTypeInfos()) { this.children.add(OrcStruct.createObjectInspector(child)); } }
private Field createField(TypeInfo typeInfo) { final Field field = new Field(); final Category category = typeInfo.getCategory(); field.category = category; field.typeInfo = typeInfo; switch (category) { case PRIMITIVE: field.primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); break; case LIST: field.children = new Field[1]; field.children[0] = createField(((ListTypeInfo) typeInfo).getListElementTypeInfo()); break; case MAP: field.children = new Field[2]; field.children[0] = createField(((MapTypeInfo) typeInfo).getMapKeyTypeInfo()); field.children[1] = createField(((MapTypeInfo) typeInfo).getMapValueTypeInfo()); break; case STRUCT: final StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; final List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); field.children = createFields(fieldTypeInfos.toArray(new TypeInfo[fieldTypeInfos.size()])); break; case UNION: final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); field.children = createFields(objectTypeInfos.toArray(new TypeInfo[objectTypeInfos.size()])); break;
OrcUnionObjectInspector(UnionTypeInfo info) { List<TypeInfo> unionChildren = info.getAllUnionObjectTypeInfos(); this.children = new ArrayList<ObjectInspector>(unionChildren.size()); for(TypeInfo child: info.getAllUnionObjectTypeInfos()) { this.children.add(OrcStruct.createObjectInspector(child)); } }
public static TypeInfo getUnionTypeInfo(List<TypeInfo> typeInfos) { TypeInfo result = cachedUnionTypeInfo.get(typeInfos); if (result == null) { result = new UnionTypeInfo(typeInfos); TypeInfo prev = cachedUnionTypeInfo.putIfAbsent(typeInfos, result); if (prev != null) { result = prev; } } return result; }
case LIST: field.children = new Field[1]; field.children[0] = createField(((ListTypeInfo) typeInfo).getListElementTypeInfo()); break; case MAP: field.children = new Field[2]; field.children[0] = createField(((MapTypeInfo) typeInfo).getMapKeyTypeInfo()); field.children[1] = createField(((MapTypeInfo) typeInfo).getMapValueTypeInfo()); break; case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); field.children = createFields(fieldTypeInfos.toArray(new TypeInfo[fieldTypeInfos.size()])); break; case UNION: UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); field.children = createFields(objectTypeInfos.toArray(new TypeInfo[objectTypeInfos.size()])); break;
private void writeUnion(FieldVector arrowVector, ColumnVector hiveVector, TypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) { final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos(); final UnionColumnVector hiveUnionVector = (UnionColumnVector) hiveVector; final ColumnVector[] hiveObjectVectors = hiveUnionVector.fields; final int tag = hiveUnionVector.tags[0]; final ColumnVector hiveObjectVector = hiveObjectVectors[tag]; final TypeInfo objectTypeInfo = objectTypeInfos.get(tag); write(arrowVector, hiveObjectVector, objectTypeInfo, size, vectorizedRowBatch, isNative); }
public static TypeInfo getUnionTypeInfo(List<TypeInfo> typeInfos) { TypeInfo result = cachedUnionTypeInfo.get(typeInfos); if (result == null) { result = new UnionTypeInfo(typeInfos); TypeInfo prev = cachedUnionTypeInfo.putIfAbsent(typeInfos, result); if (prev != null) { result = prev; } } return result; }
if (ObjectInspector.Category.PRIMITIVE.equals(typeInfo.getCategory())) { return type; else if (ObjectInspector.Category.LIST.equals(typeInfo.getCategory())) { ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; return org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "<" + escapeHiveType(listTypeInfo.getListElementTypeInfo().getTypeName()) + ">"; MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; return org.apache.hadoop.hive.serde.serdeConstants.MAP_TYPE_NAME + "<" + escapeHiveType(mapTypeInfo.getMapKeyTypeInfo().getTypeName()) + "," + escapeHiveType(mapTypeInfo.getMapValueTypeInfo().getTypeName()) + ">"; else if (ObjectInspector.Category.STRUCT.equals(typeInfo.getCategory())) { StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List<String> allStructFieldNames = structTypeInfo.getAllStructFieldNames(); List<TypeInfo> allStructFieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); StringBuilder sb = new StringBuilder(); sb.append(serdeConstants.STRUCT_TYPE_NAME + "<"); else if (ObjectInspector.Category.UNION.equals(typeInfo.getCategory())) { UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; List<TypeInfo> allUnionObjectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
private Schema createAvroUnion(TypeInfo typeInfo) { List<Schema> childSchemas = new ArrayList<Schema>(); for (TypeInfo childTypeInfo : ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos()) { final Schema childSchema = createAvroSchema(childTypeInfo); if (childSchema.getType() == Schema.Type.UNION) { childSchemas.addAll(childSchema.getTypes()); } else { childSchemas.add(childSchema); } } return Schema.createUnion(removeDuplicateNullSchemas(childSchemas)); }
public static TypeInfo getUnionTypeInfo(List<TypeInfo> typeInfos) { TypeInfo result = cachedUnionTypeInfo.get(typeInfos); if (result == null) { result = new UnionTypeInfo(typeInfos); TypeInfo prev = cachedUnionTypeInfo.putIfAbsent(typeInfos, result); if (prev != null) { result = prev; } } return result; }