/** * Check the underlying Parquet file is able to parse as Hive Decimal type. * * @param type */ protected void decimalTypeCheck(Type type) { DecimalMetadata decimalMetadata = type.asPrimitiveType().getDecimalMetadata(); if (decimalMetadata == null) { throw new UnsupportedOperationException("The underlying Parquet type cannot be able to " + "converted to Hive Decimal type: " + type); } }
/** * The decimal precision and scale is filled into decimalColumnVector. If the data in * Parquet is in decimal, the precision and scale will come in from decimalMetadata. If parquet * is not in decimal, then this call is made because HMS shows the type as decimal. So, the * precision and scale are picked from hiveType. * * @param decimalMetadata * @param decimalColumnVector */ private void fillDecimalPrecisionScale(DecimalMetadata decimalMetadata, DecimalColumnVector decimalColumnVector) { if (decimalMetadata != null) { decimalColumnVector.precision = (short) type.asPrimitiveType().getDecimalMetadata().getPrecision(); decimalColumnVector.scale = (short) type.asPrimitiveType().getDecimalMetadata().getScale(); } else if (TypeInfoUtils.getBaseName(hiveType.getTypeName()) .equalsIgnoreCase(serdeConstants.DECIMAL_TYPE_NAME)) { decimalColumnVector.precision = (short) ((DecimalTypeInfo) hiveType).getPrecision(); decimalColumnVector.scale = (short) ((DecimalTypeInfo) hiveType).getScale(); } else { throw new UnsupportedOperationException( "The underlying Parquet type cannot be converted to Hive Decimal type: " + type); } } }
case DECIMAL: decimalTypeCheck(type); int precision = type.asPrimitiveType().getDecimalMetadata().getPrecision(); int scale = type.asPrimitiveType().getDecimalMetadata().getScale(); lcv.child = new DecimalColumnVector(total, precision, scale); for (int i = 0; i < valueList.size(); i++) {
int precision = pt.asPrimitiveType().getDecimalMetadata().getPrecision(); int scale = pt.asPrimitiveType().getDecimalMetadata().getScale(); switch (pt.getPrimitiveTypeName()) { case INT32:
private void readDecimal( int total, DecimalColumnVector c, int rowId) throws IOException { DecimalMetadata decimalMetadata = type.asPrimitiveType().getDecimalMetadata(); byte[] decimalData = null; fillDecimalPrecisionScale(decimalMetadata, c); int left = total; while (left > 0) { readRepetitionAndDefinitionLevels(); if (definitionLevel >= maxDefLevel) { decimalData = dataColumn.readDecimal(); if (dataColumn.isValid()) { c.vector[rowId].set(decimalData, c.scale); c.isNull[rowId] = false; c.isRepeating = c.isRepeating && (c.vector[0] == c.vector[rowId]); } else { setNullValue(c, rowId); } } else { setNullValue(c, rowId); } rowId++; left--; } }
final short scale = (short) parquetType.asPrimitiveType().getDecimalMetadata().getScale(); return isDict ? new TypesFromDecimalPageReader(dictionary, length, scale, hivePrecision, hiveScale) : new TypesFromDecimalPageReader(valuesReader, length, scale, hivePrecision, hiveScale);
DecimalMetadata decimalMetadata = type.asPrimitiveType().getDecimalMetadata(); DecimalColumnVector decimalColumnVector = ((DecimalColumnVector) column); byte[] decimalData = null;
@Override PrimitiveStringifier stringifier(PrimitiveType type) { return PrimitiveStringifier.createDecimalStringifier(type.getDecimalMetadata().getScale()); } },
public static Optional<Type> createDecimalType(RichColumnDescriptor descriptor) { if (descriptor.getPrimitiveType().getOriginalType() != DECIMAL) { return Optional.empty(); } return Optional.of(createDecimalType(descriptor.getPrimitiveType().getDecimalMetadata())); }
public static Optional<Type> createDecimalType(RichColumnDescriptor descriptor) { if (descriptor.getPrimitiveType().getOriginalType() != DECIMAL) { return Optional.empty(); } return Optional.of(createDecimalType(descriptor.getPrimitiveType().getDecimalMetadata())); }
/** * {@inheritDoc} */ @Override public void writeToStringBuilder(StringBuilder sb, String indent) { sb.append(indent) .append(getRepetition().name().toLowerCase(Locale.ENGLISH)) .append(" ") .append(primitive.name().toLowerCase()); if (primitive == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { sb.append("(" + length + ")"); } sb.append(" ").append(getName()); if (getOriginalType() != null) { sb.append(" (").append(getOriginalType()); DecimalMetadata meta = getDecimalMetadata(); if (meta != null) { sb.append("(") .append(meta.getPrecision()) .append(",") .append(meta.getScale()) .append(")"); } sb.append(")"); } if (getId() != null) { sb.append(" = ").append(getId()); } }
private ColTypeInfo getColTypeInfo(MessageType schema, Type type, String[] path, int depth) { if (type.isPrimitive()) { PrimitiveType primitiveType = (PrimitiveType) type; int precision = 0; int scale = 0; if (primitiveType.getDecimalMetadata() != null) { precision = primitiveType.getDecimalMetadata().getPrecision(); scale = primitiveType.getDecimalMetadata().getScale(); } int repetitionLevel = schema.getMaxRepetitionLevel(path); int definitionLevel = schema.getMaxDefinitionLevel(path); return new ColTypeInfo(type.getOriginalType(), precision, scale, repetitionLevel, definitionLevel); } Type t = ((GroupType) type).getType(path[depth]); return getColTypeInfo(schema, t, path, depth + 1); }
int scale = type.getDecimalMetadata().getScale(); switch (type.getPrimitiveTypeName()) { case INT32:
private static Type getInt32Type(RichColumnDescriptor descriptor) { OriginalType originalType = descriptor.getPrimitiveType().getOriginalType(); if (originalType == null) { return INTEGER; } switch (originalType) { case DECIMAL: return createDecimalType(descriptor.getPrimitiveType().getDecimalMetadata()); case DATE: return DateType.DATE; default: return INTEGER; } } }
/** * Converts a primitive Parquet {@link Type} to a Flink {@link InternalType}. */ private InternalType convertPrimitiveType(PrimitiveType primitiveType) { PrimitiveType.PrimitiveTypeName typeName = primitiveType.getPrimitiveTypeName(); OriginalType originalType = primitiveType.getOriginalType(); DecimalMetadata decimalMetadata = primitiveType.getDecimalMetadata(); switch (typeName) { case BOOLEAN: return DataTypes.BOOLEAN; case FLOAT: return DataTypes.FLOAT; case DOUBLE: return DataTypes.DOUBLE; case INT32: return convertOriginalType_INT32(originalType, decimalMetadata); case INT64: return convertOriginalType_INT64(originalType, decimalMetadata); case BINARY: return convertOriginalType_BINARY(originalType, decimalMetadata); case INT96: case FIXED_LEN_BYTE_ARRAY: default: throw new UnsupportedOperationException(typeName + " is not supported"); } }
@Override public void visit(PrimitiveType primitiveType) { SchemaElement element = new SchemaElement(primitiveType.getName()); element.setRepetition_type(toParquetRepetition(primitiveType.getRepetition())); element.setType(getType(primitiveType.getPrimitiveTypeName())); if (primitiveType.getOriginalType() != null) { element.setConverted_type(getConvertedType(primitiveType.getOriginalType())); } if (primitiveType.getDecimalMetadata() != null) { element.setPrecision(primitiveType.getDecimalMetadata().getPrecision()); element.setScale(primitiveType.getDecimalMetadata().getScale()); } if (primitiveType.getTypeLength() > 0) { element.setType_length(primitiveType.getTypeLength()); } if (primitiveType.getId() != null) { element.setField_id(primitiveType.getId().intValue()); } result.add(element); }
@Override public void visit(PrimitiveType primitiveType) { SchemaElement element = new SchemaElement(primitiveType.getName()); element.setRepetition_type(toParquetRepetition(primitiveType.getRepetition())); element.setType(getType(primitiveType.getPrimitiveTypeName())); if (primitiveType.getLogicalTypeAnnotation() != null) { element.setConverted_type(convertToConvertedType(primitiveType.getLogicalTypeAnnotation())); element.setLogicalType(convertToLogicalType(primitiveType.getLogicalTypeAnnotation())); } if (primitiveType.getDecimalMetadata() != null) { element.setPrecision(primitiveType.getDecimalMetadata().getPrecision()); element.setScale(primitiveType.getDecimalMetadata().getScale()); } if (primitiveType.getTypeLength() > 0) { element.setType_length(primitiveType.getTypeLength()); } if (primitiveType.getId() != null) { element.setField_id(primitiveType.getId().intValue()); } result.add(element); }
private PrimitiveConverter newConverter(int colIdx, byte vecType, PrimitiveType parquetType) { switch (vecType) { case Vec.T_BAD: case Vec.T_CAT: case Vec.T_STR: case Vec.T_UUID: case Vec.T_TIME: if (OriginalType.TIMESTAMP_MILLIS.equals(parquetType.getOriginalType()) || parquetType.getPrimitiveTypeName().equals(PrimitiveType.PrimitiveTypeName.INT96)) { return new TimestampConverter(colIdx, _writer); } else { boolean dictSupport = parquetType.getOriginalType() == OriginalType.UTF8 || parquetType.getOriginalType() == OriginalType.ENUM; return new StringConverter(_writer, colIdx, dictSupport); } case Vec.T_NUM: if (OriginalType.DECIMAL.equals(parquetType.getOriginalType())) return new DecimalConverter(colIdx, parquetType.getDecimalMetadata().getScale(), _writer); else return new NumberConverter(colIdx, _writer); default: throw new UnsupportedOperationException("Unsupported type " + vecType); } }
/** * Changes the list inner '$data$' vector name to 'element' in the schema */ private Type renameChildTypeToElement(Type childType) { if (childType.isPrimitive()) { PrimitiveType childPrimitiveType = childType.asPrimitiveType(); return new PrimitiveType(childType.getRepetition(), childPrimitiveType.getPrimitiveTypeName(), childPrimitiveType.getTypeLength(), "element", childPrimitiveType.getOriginalType(), childPrimitiveType.getDecimalMetadata(), null); } else { GroupType childGroupType = childType.asGroupType(); return new GroupType(childType.getRepetition(), "element", childType.getOriginalType(), childGroupType.getFields()); } }
return Types.StringType.get(); case DECIMAL: DecimalMetadata decimal = primitive.getDecimalMetadata(); return Types.DecimalType.of( decimal.getPrecision(), decimal.getScale());