public static PrimitiveConverter getNewConverter(final PrimitiveType type, final int index, final ConverterParent parent, TypeInfo hiveTypeInfo) { if (type.isPrimitive() && (type.asPrimitiveType().getPrimitiveTypeName().equals(PrimitiveType.PrimitiveTypeName.INT96))) { //TODO- cleanup once parquet support Timestamp type annotation. return ETypeConverter.ETIMESTAMP_CONVERTER.getConverter(type, index, parent, hiveTypeInfo); } if (OriginalType.DECIMAL == type.getOriginalType()) { return EDECIMAL_CONVERTER.getConverter(type, index, parent, hiveTypeInfo); } else if (OriginalType.UTF8 == type.getOriginalType()) { return ESTRING_CONVERTER.getConverter(type, index, parent, hiveTypeInfo); } else if (OriginalType.DATE == type.getOriginalType()) { return EDATE_CONVERTER.getConverter(type, index, parent, hiveTypeInfo); } Class<?> javaType = type.getPrimitiveTypeName().javaType; for (final ETypeConverter eConverter : values()) { if (eConverter.getType() == javaType) { return eConverter.getConverter(type, index, parent, hiveTypeInfo); } } throw new IllegalArgumentException("Converter not found ... for type : " + type); }
private Object getRecordValue(Object value, Type type) { String primitive = type.asPrimitiveType().getPrimitiveTypeName().toString();
/** * {@inheritDoc} */ @Override public void writeToStringBuilder(StringBuilder sb, String indent) { sb.append(indent) .append(getRepetition().name().toLowerCase(Locale.ENGLISH)) .append(" ") .append(primitive.name().toLowerCase()); if (primitive == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { sb.append("(" + length + ")"); } sb.append(" ").append(getName()); if (getOriginalType() != null) { sb.append(" (").append(getOriginalType()); DecimalMetadata meta = getDecimalMetadata(); if (meta != null) { sb.append("(") .append(meta.getPrecision()) .append(",") .append(meta.getScale()) .append(")"); } sb.append(")"); } if (getId() != null) { sb.append(" = ").append(getId()); } }
return type.getPrimitiveTypeName().convert(new PrimitiveType.PrimitiveTypeNameConverter<TypeMapping, RuntimeException>() {
private void bind(PrimitiveTypeName type) { binding = type.convert(new PrimitiveTypeNameConverter<Binding, RuntimeException>() { @Override public Binding convertFLOAT(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
throw new ExecutionSetupException("Unsupported dictionary column type " + descriptor.getType().name() );
private void bind(PrimitiveTypeName type) { binding = type.convert(new PrimitiveTypeNameConverter<Binding, RuntimeException>() { @Override public Binding convertFLOAT(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
asPrimitive.getPrimitiveTypeName(); final OriginalType annotation = parquetType.getOriginalType(); Schema schema = parquetPrimitiveTypeName.convert( new PrimitiveType.PrimitiveTypeNameConverter<Schema, RuntimeException>() { @Override !keyType.asPrimitiveType().getPrimitiveTypeName().equals(PrimitiveTypeName.BINARY) || !keyType.getOriginalType().equals(OriginalType.UTF8)) { throw new IllegalArgumentException("Map key type must be binary (UTF8): "
return new ParquetFixedWidthDictionaryReaders.DictionaryFixedBinaryReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarBinaryVector) v, schemaElement); default: throw new ExecutionSetupException("Unsupported dictionary column type " + descriptor.getType().name() );
return new NullableFixedByteAlignedReaders.NullableDictionaryFloat8Reader(parentReader, columnDescriptor, columnChunkMetaData, fixedLength, (NullableFloat8Vector)valueVec, schemaElement); default: throw new ExecutionSetupException("Unsupported nullable column type " + columnDescriptor.getType().name() );
if (schema.containsField(columnSchema.getName())) { if (!schema.getType(columnSchema.getName()).asPrimitiveType().getPrimitiveTypeName() .equals(getTypeName(columnSchema.getType()))) { throw new IllegalArgumentException("The column type " + getTypeName(columnSchema.getType()) + " does not exist in Parquet schema"); while (fields.hasNext()) { ColumnDescriptor colDesc = fields.next(); if (colDesc.getType().equals(PrimitiveTypeName.INT96)) { throw new IllegalArgumentException("Column type not supported in Kudu: " + colDesc);
return new NullableFixedByteAlignedReaders.NullableDictionaryFloat8Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (Float8Vector)valueVec, schemaElement); default: throw new ExecutionSetupException("Unsupported nullable column type " + columnDescriptor.getType().name() );
fieldType.asPrimitiveType().getPrimitiveTypeName(); final OriginalType originalType = fieldType.getOriginalType(); return parquetPrimitiveTypeName.convert( new PrimitiveType.PrimitiveTypeNameConverter<Column, RuntimeException>() { @Override
fieldType.asPrimitiveType().getPrimitiveTypeName(); final OriginalType originalType = fieldType.getOriginalType(); return parquetPrimitiveTypeName.convert( new PrimitiveType.PrimitiveTypeNameConverter<Column, RuntimeException>() { @Override
@Override protected Type union(Type toMerge, boolean strict) { if (!toMerge.isPrimitive()) { reportSchemaMergeError(toMerge); } if (strict) { // Can't merge primitive fields of different type names or different original types if (!primitive.equals(toMerge.asPrimitiveType().getPrimitiveTypeName()) || !Objects.equals(getLogicalTypeAnnotation(), toMerge.getLogicalTypeAnnotation())) { reportSchemaMergeError(toMerge); } // Can't merge FIXED_LEN_BYTE_ARRAY fields of different lengths int toMergeLength = toMerge.asPrimitiveType().getTypeLength(); if (primitive == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY && length != toMergeLength) { reportSchemaMergeError(toMerge); } // Can't merge primitive fields with different column orders if (!columnOrder().equals(toMerge.asPrimitiveType().columnOrder())) { reportSchemaMergeErrorWithColumnOrder(toMerge); } } Types.PrimitiveBuilder<PrimitiveType> builder = Types.primitive(primitive, toMerge.getRepetition()); if (PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY == primitive) { builder.length(length); } return builder.as(getLogicalTypeAnnotation()).named(getName()); }
@Override protected Type union(Type toMerge, boolean strict) { if (!toMerge.isPrimitive()) { reportSchemaMergeError(toMerge); } if (strict) { // Can't merge primitive fields of different type names or different original types if (!primitive.equals(toMerge.asPrimitiveType().getPrimitiveTypeName()) || getOriginalType() != toMerge.getOriginalType()) { reportSchemaMergeError(toMerge); } // Can't merge FIXED_LEN_BYTE_ARRAY fields of different lengths int toMergeLength = toMerge.asPrimitiveType().getTypeLength(); if (primitive == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY && length != toMergeLength) { reportSchemaMergeError(toMerge); } // Can't merge primitive fields with different column orders if (!columnOrder().equals(toMerge.asPrimitiveType().columnOrder())) { reportSchemaMergeErrorWithColumnOrder(toMerge); } } Types.PrimitiveBuilder<PrimitiveType> builder = Types.primitive(primitive, toMerge.getRepetition()); if (PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY == primitive) { builder.length(length); } return builder.as(getOriginalType()).named(getName()); }
private PrimitiveConverter newConverter(int colIdx, byte vecType, PrimitiveType parquetType) { switch (vecType) { case Vec.T_BAD: case Vec.T_CAT: case Vec.T_STR: case Vec.T_UUID: case Vec.T_TIME: if (OriginalType.TIMESTAMP_MILLIS.equals(parquetType.getOriginalType()) || parquetType.getPrimitiveTypeName().equals(PrimitiveType.PrimitiveTypeName.INT96)) { return new TimestampConverter(colIdx, _writer); } else { boolean dictSupport = parquetType.getOriginalType() == OriginalType.UTF8 || parquetType.getOriginalType() == OriginalType.ENUM; return new StringConverter(_writer, colIdx, dictSupport); } case Vec.T_NUM: if (OriginalType.DECIMAL.equals(parquetType.getOriginalType())) return new DecimalConverter(colIdx, parquetType.getDecimalMetadata().getScale(), _writer); else return new NumberConverter(colIdx, _writer); default: throw new UnsupportedOperationException("Unsupported type " + vecType); } }
/** * {@inheritDoc} */ @Override public void writeToStringBuilder(StringBuilder sb, String indent) { sb.append(indent) .append(getRepetition().name().toLowerCase(Locale.ENGLISH)) .append(" ") .append(primitive.name().toLowerCase()); if (primitive == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { sb.append("(" + length + ")"); } sb.append(" ").append(getName()); if (getLogicalTypeAnnotation() != null) { // TODO: should we print decimal metadata too? sb.append(" (").append(getLogicalTypeAnnotation().toString()).append(")"); } if (getId() != null) { sb.append(" = ").append(getId()); } }
@Override protected void readValue(BlockBuilder blockBuilder, Type type) { if (definitionLevel == columnDescriptor.getMaxDefinitionLevel()) { long decimalValue; // When decimals are encoded with primitive types Parquet stores unscaled values if (columnDescriptor.getType().equals(INT32)) { decimalValue = valuesReader.readInteger(); } else if (columnDescriptor.getType().equals(INT64)) { decimalValue = valuesReader.readLong(); } else { decimalValue = getShortDecimalValue(valuesReader.readBytes().getBytes()); } type.writeLong(blockBuilder, decimalValue); } else if (isValueNull()) { blockBuilder.appendNull(); } }
@Override protected void readValue(BlockBuilder blockBuilder, Type type) { if (definitionLevel == columnDescriptor.getMaxDefinitionLevel()) { long decimalValue; // When decimals are encoded with primitive types Parquet stores unscaled values if (columnDescriptor.getType().equals(INT32)) { decimalValue = valuesReader.readInteger(); } else if (columnDescriptor.getType().equals(INT64)) { decimalValue = valuesReader.readLong(); } else { decimalValue = getShortDecimalValue(valuesReader.readBytes().getBytes()); } type.writeLong(blockBuilder, decimalValue); } else if (isValueNull()) { blockBuilder.appendNull(); } }