Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType);
@Override public int hashCode() { int c = repetition.hashCode(); c = 31 * c + name.hashCode(); if (logicalTypeAnnotation != null) { c = 31 * c + logicalTypeAnnotation.hashCode(); } if (id != null) { c = 31 * c + id.hashCode(); } return c; }
int fieldId = primitive.getId().intValue(); Schema avroSchema = AvroSchemaUtil.convert(schema.findType(fieldId)); return new FixedReader(desc, avroSchema);
Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType);
Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType);
Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType);
private boolean eval(MessageType fileSchema, BlockMetaData rowGroup, DictionaryPageReadStore dictionaries) { this.dictionaries = dictionaries; this.dictCache = Maps.newHashMap(); this.isFallback = Maps.newHashMap(); this.cols = Maps.newHashMap(); this.conversions = Maps.newHashMap(); for (ColumnDescriptor desc : fileSchema.getColumns()) { PrimitiveType colType = fileSchema.getType(desc.getPath()).asPrimitiveType(); if (colType.getId() != null) { int id = colType.getId().intValue(); cols.put(id, desc); conversions.put(id, converterFromParquet(colType)); } } for (ColumnChunkMetaData meta : rowGroup.getColumns()) { PrimitiveType colType = fileSchema.getType(meta.getPath().toArray()).asPrimitiveType(); if (colType.getId() != null) { int id = colType.getId().intValue(); isFallback.put(id, hasNonDictionaryPages(meta)); } } return ExpressionVisitors.visit(expr, this); }
@Override protected Type build(String name) { Preconditions.checkState(logicalTypeAnnotation == null, "MAP is already a logical type and can't be changed."); if (keyType == null) { keyType = STRING_KEY; } GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.MAP); if (id != null) { builder.id(id.intValue()); } if (valueType != null) { return builder .repeatedGroup().addFields(keyType, valueType).named("map") .named(name); } else { return builder .repeatedGroup().addFields(keyType).named("map") .named(name); } } }
@Override protected Type build(String name) { Preconditions.checkState(originalType == null, "MAP is already a logical type and can't be changed."); if (keyType == null) { keyType = STRING_KEY; } GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.MAP); if (id != null) { builder.id(id.intValue()); } if (valueType != null) { return builder .repeatedGroup().addFields(keyType, valueType).named("map") .named(name); } else { return builder .repeatedGroup().addFields(keyType).named("map") .named(name); } } }
@Override public void visit(PrimitiveType primitiveType) { SchemaElement element = new SchemaElement(primitiveType.getName()); element.setRepetition_type(toParquetRepetition(primitiveType.getRepetition())); element.setType(getType(primitiveType.getPrimitiveTypeName())); if (primitiveType.getLogicalTypeAnnotation() != null) { element.setConverted_type(convertToConvertedType(primitiveType.getLogicalTypeAnnotation())); element.setLogicalType(convertToLogicalType(primitiveType.getLogicalTypeAnnotation())); } if (primitiveType.getDecimalMetadata() != null) { element.setPrecision(primitiveType.getDecimalMetadata().getPrecision()); element.setScale(primitiveType.getDecimalMetadata().getScale()); } if (primitiveType.getTypeLength() > 0) { element.setType_length(primitiveType.getTypeLength()); } if (primitiveType.getId() != null) { element.setField_id(primitiveType.getId().intValue()); } result.add(element); }
@Override public void visit(PrimitiveType primitiveType) { SchemaElement element = new SchemaElement(primitiveType.getName()); element.setRepetition_type(toParquetRepetition(primitiveType.getRepetition())); element.setType(getType(primitiveType.getPrimitiveTypeName())); if (primitiveType.getOriginalType() != null) { element.setConverted_type(getConvertedType(primitiveType.getOriginalType())); } if (primitiveType.getDecimalMetadata() != null) { element.setPrecision(primitiveType.getDecimalMetadata().getPrecision()); element.setScale(primitiveType.getDecimalMetadata().getScale()); } if (primitiveType.getTypeLength() > 0) { element.setType_length(primitiveType.getTypeLength()); } if (primitiveType.getId() != null) { element.setField_id(primitiveType.getId().intValue()); } result.add(element); }
private boolean eval(MessageType fileSchema, BlockMetaData rowGroup) { if (rowGroup.getRowCount() <= 0) { return ROWS_CANNOT_MATCH; } this.stats = Maps.newHashMap(); this.valueCounts = Maps.newHashMap(); this.conversions = Maps.newHashMap(); for (ColumnChunkMetaData col : rowGroup.getColumns()) { PrimitiveType colType = fileSchema.getType(col.getPath().toArray()).asPrimitiveType(); if (colType.getId() != null) { int id = colType.getId().intValue(); stats.put(id, col.getStatistics()); valueCounts.put(id, col.getValueCount()); conversions.put(id, converterFromParquet(colType)); } } return ExpressionVisitors.visit(expr, this); }
@Override public void visit(GroupType groupType) { SchemaElement element = new SchemaElement(groupType.getName()); element.setRepetition_type(toParquetRepetition(groupType.getRepetition())); if (groupType.getLogicalTypeAnnotation() != null) { element.setConverted_type(convertToConvertedType(groupType.getLogicalTypeAnnotation())); element.setLogicalType(convertToLogicalType(groupType.getLogicalTypeAnnotation())); } if (groupType.getId() != null) { element.setField_id(groupType.getId().intValue()); } visitChildren(result, groupType, element); }
/** * @param id the field id * @return a new GroupType with the same fields and a new id */ @Override public GroupType withId(int id) { return new GroupType(getRepetition(), getName(), getLogicalTypeAnnotation(), fields, new ID(id)); }
/** * adds an id annotation to the type being built. * <p> * ids are used to capture the original id when converting from models using ids (thrift, protobufs) * * @param id the id of the field * @return this builder for method chaining */ public THIS id(int id) { this.id = new ID(id); return self(); }
@Override protected Type build(String name) { Preconditions.checkState(logicalTypeAnnotation == null, "LIST is already the logical type and can't be changed"); Preconditions.checkNotNull(elementType, "List element type"); GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.LIST); if (id != null) { builder.id(id.intValue()); } return builder .repeatedGroup().addFields(elementType).named("list") .named(name); }
/** * adds an id annotation to the type being built. * <p> * ids are used to capture the original id when converting from models using ids (thrift, protobufs) * * @param id the id of the field * @return this builder for method chaining */ public THIS id(int id) { this.id = new ID(id); return self(); }
@Override public void visit(MessageType messageType) { SchemaElement element = new SchemaElement(messageType.getName()); if (messageType.getId() != null) { element.setField_id(messageType.getId().intValue()); } visitChildren(result, messageType.asGroupType(), element); }
/** * @param id the field id * @return a new PrimitiveType with the same fields and a new id */ @Override public PrimitiveType withId(int id) { return new PrimitiveType(getRepetition(), primitive, length, getName(), getLogicalTypeAnnotation(), new ID(id), columnOrder); }
@Override protected Type build(String name) { Preconditions.checkState(originalType == null, "LIST is already the logical type and can't be changed"); Preconditions.checkNotNull(elementType, "List element type"); GroupBuilder<GroupType> builder = buildGroup(repetition).as(OriginalType.LIST); if (id != null) { builder.id(id.intValue()); } return builder .repeatedGroup().addFields(elementType).named("list") .named(name); }