@SuppressWarnings("deprecation") private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo) { final TypeInfo subType = typeInfo.getListElementTypeInfo(); return new GroupType(Repetition.OPTIONAL, name, OriginalType.LIST, new GroupType(Repetition.REPEATED, ParquetHiveSerDe.ARRAY.toString(), convertType("array_element", subType))); }
private static GroupType buildProjectedGroupType( GroupType originalType, List<Type> types) { if (types == null || types.isEmpty()) { return null; } return new GroupType(originalType.getRepetition(), originalType.getName(), types); }
private static GroupType convertStructType(final String name, final StructTypeInfo typeInfo) { final List<String> columnNames = typeInfo.getAllStructFieldNames(); final List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos(); return new GroupType(Repetition.OPTIONAL, name, convertTypes(columnNames, columnTypes)); }
Text name = new Text(subFieldName); if (name.equals(ParquetHiveSerDe.ARRAY) || name.equals(ParquetHiveSerDe.LIST)) { subFieldType = new GroupType(Repetition.REPEATED, subFieldName, getProjectedType(elemType, subFieldType.asGroupType().getType(0))); } else {
private static GroupType listWrapper(final String name, final OriginalType originalType, final Type elementType, final Repetition repetition) { return new GroupType(repetition, name, originalType, elementType); } }
private static GroupType listWrapper(final String name, final OriginalType originalType, final Type elementType, final Repetition repetition) { return new GroupType(repetition, name, originalType, elementType); } }
private static GroupType listWrapper(final String name, final OriginalType originalType, final GroupType groupType) { return new GroupType(Repetition.OPTIONAL, name, originalType, groupType); } }
public RelationWriteSupport(boolean excludeMetadata) { super(excludeMetadata); memberIdType = new PrimitiveType(REQUIRED, INT64, "id"); memberRoleType = new PrimitiveType(REQUIRED, BINARY, "role"); memberTypeType = new PrimitiveType(REQUIRED, BINARY, "type"); membersType = new GroupType(REPEATED, "members", memberIdType, memberRoleType, memberTypeType); }
private static GroupType listWrapper(Repetition repetition, String alias, OriginalType originalType, Type nested) { if (!nested.isRepetition(Repetition.REPEATED)) { throw new IllegalArgumentException("Nested type should be repeated: " + nested); } return new GroupType(repetition, alias, originalType, nested); }
private static Type getType(String[] pathSegments, int depth, MessageType schema) { Type type = schema.getType(Arrays.copyOfRange(pathSegments, 0, depth + 1)); if (depth + 1 == pathSegments.length) { return type; } else { Preconditions.checkState(!type.isPrimitive()); return new GroupType(type.getRepetition(), type.getName(), type.getOriginalType(), getType(pathSegments, depth + 1, schema)); } }
/** * @param newFields a list of types to use as fields in a copy of this group * @return a group with the same attributes and new fields. */ public GroupType withNewFields(List<Type> newFields) { return new GroupType(getRepetition(), getName(), getOriginalType(), newFields, getId()); }
/** * @param newFields a list of types to use as fields in a copy of this group * @return a group with the same attributes and new fields. */ public GroupType withNewFields(List<Type> newFields) { return new GroupType(getRepetition(), getName(), getLogicalTypeAnnotation(), newFields, getId()); }
private static Type getType(String[] pathSegments, int depth, MessageType schema) { Type type = schema.getType(Arrays.copyOfRange(pathSegments, 0, depth + 1)); if (depth + 1 == pathSegments.length) { return type; } else { Preconditions.checkState(!type.isPrimitive()); return new GroupType(type.getRepetition(), type.getName(), getType(pathSegments, depth + 1, schema)); } }
/** * @param id the field id * @return a new GroupType with the same fields and a new id */ @Override public GroupType withId(int id) { return new GroupType(getRepetition(), getName(), getOriginalType(), fields, new ID(id)); }
@Test public void testParquetTupleDomainPrimitiveArray() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_array", HiveType.valueOf("array<int>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty()); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(INTEGER)))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_array", new GroupType(REPEATED, "bag", new PrimitiveType(OPTIONAL, INT32, "array_element")))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo) { final TypeInfo subType = typeInfo.getListElementTypeInfo(); return listWrapper(name, OriginalType.LIST, new GroupType(Repetition.REPEATED, ParquetHiveSerDe.ARRAY.toString(), convertType("array_element", subType))); }
private static GroupType convertStructType(final String name, final StructTypeInfo typeInfo, final Repetition repetition) { final List<String> columnNames = typeInfo.getAllStructFieldNames(); final List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos(); return new GroupType(repetition, name, convertTypes(columnNames, columnTypes)); }
private static GroupType convertStructType(final String name, final StructTypeInfo typeInfo, final Repetition repetition) { final List<String> columnNames = typeInfo.getAllStructFieldNames(); final List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos(); return new GroupType(repetition, name, convertTypes(columnNames, columnTypes)); }
private static GroupType convertStructType(final String name, final StructTypeInfo typeInfo) { final List<String> columnNames = typeInfo.getAllStructFieldNames(); final List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos(); return new GroupType(Repetition.OPTIONAL, name, convertTypes(columnNames, columnTypes)); }
@Override protected Type union(Type toMerge, boolean strict) { if (toMerge.isPrimitive()) { throw new IncompatibleSchemaModificationException("can not merge primitive type " + toMerge + " into group type " + this); } return new GroupType(toMerge.getRepetition(), getName(), toMerge.getLogicalTypeAnnotation(), mergeFields(toMerge.asGroupType()), getId()); }