Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType);
Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType);
Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType);
int fieldId = primitive.getId().intValue(); Schema avroSchema = AvroSchemaUtil.convert(schema.findType(fieldId)); return new FixedReader(desc, avroSchema);
Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType);
private boolean eval(MessageType fileSchema, BlockMetaData rowGroup, DictionaryPageReadStore dictionaries) { this.dictionaries = dictionaries; this.dictCache = Maps.newHashMap(); this.isFallback = Maps.newHashMap(); this.cols = Maps.newHashMap(); this.conversions = Maps.newHashMap(); for (ColumnDescriptor desc : fileSchema.getColumns()) { PrimitiveType colType = fileSchema.getType(desc.getPath()).asPrimitiveType(); if (colType.getId() != null) { int id = colType.getId().intValue(); cols.put(id, desc); conversions.put(id, converterFromParquet(colType)); } } for (ColumnChunkMetaData meta : rowGroup.getColumns()) { PrimitiveType colType = fileSchema.getType(meta.getPath().toArray()).asPrimitiveType(); if (colType.getId() != null) { int id = colType.getId().intValue(); isFallback.put(id, hasNonDictionaryPages(meta)); } } return ExpressionVisitors.visit(expr, this); }
@Override protected Type build(String name) { Preconditions.checkState(logicalTypeAnnotation == null, "MAP is already a logical type and can't be changed."); if (keyType == null) { keyType = STRING_KEY; } GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.MAP); if (id != null) { builder.id(id.intValue()); } if (valueType != null) { return builder .repeatedGroup().addFields(keyType, valueType).named("map") .named(name); } else { return builder .repeatedGroup().addFields(keyType).named("map") .named(name); } } }
@Override protected Type build(String name) { Preconditions.checkState(originalType == null, "MAP is already a logical type and can't be changed."); if (keyType == null) { keyType = STRING_KEY; } GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.MAP); if (id != null) { builder.id(id.intValue()); } if (valueType != null) { return builder .repeatedGroup().addFields(keyType, valueType).named("map") .named(name); } else { return builder .repeatedGroup().addFields(keyType).named("map") .named(name); } } }
@Override public void visit(PrimitiveType primitiveType) { SchemaElement element = new SchemaElement(primitiveType.getName()); element.setRepetition_type(toParquetRepetition(primitiveType.getRepetition())); element.setType(getType(primitiveType.getPrimitiveTypeName())); if (primitiveType.getLogicalTypeAnnotation() != null) { element.setConverted_type(convertToConvertedType(primitiveType.getLogicalTypeAnnotation())); element.setLogicalType(convertToLogicalType(primitiveType.getLogicalTypeAnnotation())); } if (primitiveType.getDecimalMetadata() != null) { element.setPrecision(primitiveType.getDecimalMetadata().getPrecision()); element.setScale(primitiveType.getDecimalMetadata().getScale()); } if (primitiveType.getTypeLength() > 0) { element.setType_length(primitiveType.getTypeLength()); } if (primitiveType.getId() != null) { element.setField_id(primitiveType.getId().intValue()); } result.add(element); }
@Override public void visit(PrimitiveType primitiveType) { SchemaElement element = new SchemaElement(primitiveType.getName()); element.setRepetition_type(toParquetRepetition(primitiveType.getRepetition())); element.setType(getType(primitiveType.getPrimitiveTypeName())); if (primitiveType.getOriginalType() != null) { element.setConverted_type(getConvertedType(primitiveType.getOriginalType())); } if (primitiveType.getDecimalMetadata() != null) { element.setPrecision(primitiveType.getDecimalMetadata().getPrecision()); element.setScale(primitiveType.getDecimalMetadata().getScale()); } if (primitiveType.getTypeLength() > 0) { element.setType_length(primitiveType.getTypeLength()); } if (primitiveType.getId() != null) { element.setField_id(primitiveType.getId().intValue()); } result.add(element); }
private boolean eval(MessageType fileSchema, BlockMetaData rowGroup) { if (rowGroup.getRowCount() <= 0) { return ROWS_CANNOT_MATCH; } this.stats = Maps.newHashMap(); this.valueCounts = Maps.newHashMap(); this.conversions = Maps.newHashMap(); for (ColumnChunkMetaData col : rowGroup.getColumns()) { PrimitiveType colType = fileSchema.getType(col.getPath().toArray()).asPrimitiveType(); if (colType.getId() != null) { int id = colType.getId().intValue(); stats.put(id, col.getStatistics()); valueCounts.put(id, col.getValueCount()); conversions.put(id, converterFromParquet(colType)); } } return ExpressionVisitors.visit(expr, this); }
@Override public void visit(GroupType groupType) { SchemaElement element = new SchemaElement(groupType.getName()); element.setRepetition_type(toParquetRepetition(groupType.getRepetition())); if (groupType.getLogicalTypeAnnotation() != null) { element.setConverted_type(convertToConvertedType(groupType.getLogicalTypeAnnotation())); element.setLogicalType(convertToLogicalType(groupType.getLogicalTypeAnnotation())); } if (groupType.getId() != null) { element.setField_id(groupType.getId().intValue()); } visitChildren(result, groupType, element); }
@Override protected Type build(String name) { Preconditions.checkState(originalType == null, "LIST is already the logical type and can't be changed"); Preconditions.checkNotNull(elementType, "List element type"); GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.LIST); if (id != null) { builder.id(id.intValue()); } return builder .repeatedGroup().addFields(elementType).named("list") .named(name); }
@Override protected Type build(String name) { Preconditions.checkState(logicalTypeAnnotation == null, "LIST is already the logical type and can't be changed"); Preconditions.checkNotNull(elementType, "List element type"); GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.LIST); if (id != null) { builder.id(id.intValue()); } return builder .repeatedGroup().addFields(elementType).named("list") .named(name); }
@Override public void visit(GroupType groupType) { SchemaElement element = new SchemaElement(groupType.getName()); element.setRepetition_type(toParquetRepetition(groupType.getRepetition())); if (groupType.getOriginalType() != null) { element.setConverted_type(getConvertedType(groupType.getOriginalType())); } if (groupType.getId() != null) { element.setField_id(groupType.getId().intValue()); } visitChildren(result, groupType, element); }
@Override public void visit(MessageType messageType) { SchemaElement element = new SchemaElement(messageType.getName()); if (messageType.getId() != null) { element.setField_id(messageType.getId().intValue()); } visitChildren(result, messageType.asGroupType(), element); }
@Override public void visit(MessageType messageType) { SchemaElement element = new SchemaElement(messageType.getName()); if (messageType.getId() != null) { element.setField_id(messageType.getId().intValue()); } visitChildren(result, messageType.asGroupType(), element); }
private static <T> List<T> visitFields(Types.StructType struct, GroupType group, TypeWithSchemaVisitor<T> visitor) { List<T> results = Lists.newArrayListWithExpectedSize(group.getFieldCount()); for (Type field : group.getFields()) { int id = -1; if (field.getId() != null) { id = field.getId().intValue(); } Types.NestedField iField = (struct != null && id >= 0) ? struct.field(id) : null; results.add(visitField(iField, field, visitor)); } return results; }
private int getId(org.apache.parquet.schema.Type type) { org.apache.parquet.schema.Type.ID id = type.getId(); if (id != null) { return id.intValue(); } else { return nextId(); } } }
private int getId(Type type) { Preconditions.checkNotNull(type.getId(), "Missing id for type: " + type); return type.getId().intValue(); } }