public static boolean isStructuralType(HiveType hiveType) { return hiveType.getCategory() == Category.LIST || hiveType.getCategory() == Category.MAP || hiveType.getCategory() == Category.STRUCT; }
public static TupleDomain<ColumnDescriptor> getParquetTupleDomain(Map<List<String>, RichColumnDescriptor> descriptorsByPath, TupleDomain<HiveColumnHandle> effectivePredicate) { if (effectivePredicate.isNone()) { return TupleDomain.none(); } ImmutableMap.Builder<ColumnDescriptor, Domain> predicate = ImmutableMap.builder(); for (Entry<HiveColumnHandle, Domain> entry : effectivePredicate.getDomains().get().entrySet()) { HiveColumnHandle columnHandle = entry.getKey(); // skip looking up predicates for complex types as Parquet only stores stats for primitives if (!columnHandle.getHiveType().getCategory().equals(PRIMITIVE)) { continue; } RichColumnDescriptor descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName())); if (descriptor != null) { predicate.put(descriptor, entry.getValue()); } } return TupleDomain.withColumnDomains(predicate.build()); }
private boolean canCoerceForStruct(HiveType fromHiveType, HiveType toHiveType) { if (!fromHiveType.getCategory().equals(Category.STRUCT) || !toHiveType.getCategory().equals(Category.STRUCT)) { return false; } List<String> fromFieldNames = ((StructTypeInfo) fromHiveType.getTypeInfo()).getAllStructFieldNames(); List<String> toFieldNames = ((StructTypeInfo) toHiveType.getTypeInfo()).getAllStructFieldNames(); List<HiveType> fromFieldTypes = extractStructFieldTypes(fromHiveType); List<HiveType> toFieldTypes = extractStructFieldTypes(toHiveType); // Rule: // * Fields may be added or dropped from the end. // * For all other field indices, the corresponding fields must have // the same name, and the type must be coercible. for (int i = 0; i < min(fromFieldTypes.size(), toFieldTypes.size()); i++) { if (!fromFieldNames.get(i).equals(toFieldNames.get(i))) { return false; } if (!fromFieldTypes.get(i).equals(toFieldTypes.get(i)) && !canCoerce(fromFieldTypes.get(i), toFieldTypes.get(i))) { return false; } } return true; } }
private boolean canCoerceForList(HiveType fromHiveType, HiveType toHiveType) { if (!fromHiveType.getCategory().equals(Category.LIST) || !toHiveType.getCategory().equals(Category.LIST)) { return false; } HiveType fromElementType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); return fromElementType.equals(toElementType) || canCoerce(fromElementType, toElementType); }
private boolean canCoerceForMap(HiveType fromHiveType, HiveType toHiveType) { if (!fromHiveType.getCategory().equals(Category.MAP) || !toHiveType.getCategory().equals(Category.MAP)) { return false; } HiveType fromKeyType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); return (fromKeyType.equals(toKeyType) || canCoerce(fromKeyType, toKeyType)) && (fromValueType.equals(toValueType) || canCoerce(fromValueType, toValueType)); }
public static boolean isStructuralType(HiveType hiveType) { return hiveType.getCategory() == Category.LIST || hiveType.getCategory() == Category.MAP || hiveType.getCategory() == Category.STRUCT; }
private void parseObjectColumn(int column, byte[] bytes, int start, int length) { checkState(VALID_HIVE_STRUCTURAL_CATEGORIES.contains(hiveTypes[column].getCategory()), "%s is not a valid STRUCTURAL type", hiveTypes[column]); if (length == 0) { nulls[column] = true; } else { nulls[column] = false; LazyBinaryObject<? extends ObjectInspector> lazyObject = LazyBinaryFactory.createLazyBinaryObject(fieldInspectors[column]); ByteArrayRef byteArrayRef = new ByteArrayRef(); byteArrayRef.setData(bytes); lazyObject.init(byteArrayRef, start, length); objects[column] = getBlockObject(types[column], lazyObject.getObject(), fieldInspectors[column]); } }