private TypeInfo extractListCompatibleType(TypeInfo hiveTypeInfo) { if (hiveTypeInfo != null && hiveTypeInfo instanceof ListTypeInfo) { return ((ListTypeInfo) hiveTypeInfo).getListElementTypeInfo(); } else { return hiveTypeInfo; //to handle map can read list of struct data (i.e. list<struct<key, value>> --> map<key, // value>) } }
@Override public boolean equals(Object other) { if (this == other) { return true; } if (!(other instanceof ListTypeInfo)) { return false; } return getListElementTypeInfo().equals( ((ListTypeInfo) other).getListElementTypeInfo()); }
OrcListObjectInspector(ListTypeInfo info) { child = createObjectInspector(info.getListElementTypeInfo()); }
private static int hashOfList(ListTypeInfo type, Block singleListBlock) { TypeInfo elementTypeInfo = type.getListElementTypeInfo(); int result = 0; for (int i = 0; i < singleListBlock.getPositionCount(); i++) { result = result * 31 + hash(elementTypeInfo, singleListBlock, i); } return result; }
public static boolean lazyCompareList(ListTypeInfo listTypeInfo, List<Object> list, List<Object> expectedList) { TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo(); final int size = list.size(); for (int i = 0; i < size; i++) { Object lazyEleObj = list.get(i); Object expectedEleObj = expectedList.get(i); if (!lazyCompare(elementTypeInfo, lazyEleObj, expectedEleObj)) { throw new RuntimeException("List element deserialized value does not match elementTypeInfo " + elementTypeInfo.toString()); } } return true; }
public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType); }
private Schema createAvroArray(TypeInfo typeInfo) { ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; Schema listSchema = createAvroSchema(listTypeInfo.getListElementTypeInfo()); return Schema.createArray(listSchema); }
public static RelDataType convert(ListTypeInfo lstType, RelDataTypeFactory dtFactory) throws CalciteSemanticException { RelDataType elemType = convert(lstType.getListElementTypeInfo(), dtFactory); return dtFactory.createArrayType(elemType, -1); }
public static RelDataType convert(ListTypeInfo lstType, RelDataTypeFactory dtFactory) throws CalciteSemanticException { RelDataType elemType = convert(lstType.getListElementTypeInfo(), dtFactory); return dtFactory.createArrayType(elemType, -1); }
public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType, BridgingRecordCursor bridgingRecordCursor) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); this.bridgingRecordCursor = requireNonNull(bridgingRecordCursor, "bridgingRecordCursor is null"); HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); this.fromElementType = fromElementHiveType.getType(typeManager); this.toType = toHiveType.getType(typeManager); this.toElementType = toElementHiveType.getType(typeManager); this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType, bridgingRecordCursor); this.pageBuilder = elementCoercer == null ? null : new PageBuilder(ImmutableList.of(toType)); }
private Object serializeList(ListTypeInfo typeInfo, ListObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { List<?> list = fieldOI.getList(structFieldData); List<Object> deserialized = new GenericData.Array<Object>(list.size(), schema); TypeInfo listElementTypeInfo = typeInfo.getListElementTypeInfo(); ObjectInspector listElementObjectInspector = fieldOI.getListElementObjectInspector(); Schema elementType = schema.getElementType(); for(int i = 0; i < list.size(); i++) { deserialized.add(i, serialize(listElementTypeInfo, listElementObjectInspector, list.get(i), elementType)); } return deserialized; }
@SuppressWarnings("deprecation") private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo) { final TypeInfo subType = typeInfo.getListElementTypeInfo(); return new GroupType(Repetition.OPTIONAL, name, OriginalType.LIST, new GroupType(Repetition.REPEATED, ParquetHiveSerDe.ARRAY.toString(), convertType("array_element", subType))); }
@SuppressWarnings("deprecation") private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo) { final TypeInfo subType = typeInfo.getListElementTypeInfo(); return new GroupType(Repetition.OPTIONAL, name, OriginalType.LIST, new GroupType(Repetition.REPEATED, ParquetHiveSerDe.ARRAY.toString(), convertType("array_element", subType))); }
private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo, final Repetition repetition) { final TypeInfo subType = typeInfo.getListElementTypeInfo(); return listWrapper(name, OriginalType.LIST, convertType("array", subType, Repetition.REPEATED), repetition); }
private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo, final Repetition repetition) { final TypeInfo subType = typeInfo.getListElementTypeInfo(); return listWrapper(name, OriginalType.LIST, convertType("array_element", subType, Repetition.REPEATED), repetition); }
public static boolean isSupportedType(TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: return getPrimitiveType((PrimitiveTypeInfo) typeInfo) != null; case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; return isSupportedType(mapTypeInfo.getMapKeyTypeInfo()) && isSupportedType(mapTypeInfo.getMapValueTypeInfo()); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; return isSupportedType(listTypeInfo.getListElementTypeInfo()); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; return structTypeInfo.getAllStructFieldTypeInfos().stream() .allMatch(HiveType::isSupportedType); } return false; }
private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo) { final TypeInfo subType = typeInfo.getListElementTypeInfo(); return listWrapper(name, OriginalType.LIST, new GroupType(Repetition.REPEATED, ParquetHiveSerDe.ARRAY.toString(), convertType("array_element", subType))); }
private boolean canCoerceForList(HiveType fromHiveType, HiveType toHiveType) { if (!fromHiveType.getCategory().equals(Category.LIST) || !toHiveType.getCategory().equals(Category.LIST)) { return false; } HiveType fromElementType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); return fromElementType.equals(toElementType) || canCoerce(fromElementType, toElementType); }
private static boolean isWritableType(TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); return isWritablePrimitiveType(primitiveCategory); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; return isWritableType(mapTypeInfo.getMapKeyTypeInfo()) && isWritableType(mapTypeInfo.getMapValueTypeInfo()); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; return isWritableType(listTypeInfo.getListElementTypeInfo()); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; return structTypeInfo.getAllStructFieldTypeInfos().stream().allMatch(HiveWriteUtils::isWritableType); } return false; }
private static ExprNodeDesc getListIndexNode(ExprNodeDesc node, ExprNodeDesc index) throws Exception { ArrayList<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(2); children.add(node); children.add(index); return new ExprNodeGenericFuncDesc(((ListTypeInfo) node.getTypeInfo()) .getListElementTypeInfo(), FunctionRegistry.getGenericUDFForIndex(), children); }