private void writeMinMax(DataOutput out) throws IOException { DataType dataType = columnSpec.getSchemaDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { out.writeByte((byte) getMaxValue()); out.writeFloat(0f); // unique value is obsoleted, maintain for compatibility } else if (DataTypes.isDecimal(dataType)) { byte[] maxAsBytes = getMaxAsBytes(columnSpec.getSchemaDataType()); byte[] minAsBytes = getMinAsBytes(columnSpec.getSchemaDataType()); byte[] unique = DataTypeUtil.bigDecimalToByte(BigDecimal.ZERO); out.writeShort((short) maxAsBytes.length);
.getDataType().getName() + " Destination data type: " + targetDataType .getName() + " for the column: " + noDictDimensionPages[noDictIndex] .getColumnSpec().getFieldName() + " having encoding type: " + columnPageEncoder.getEncodingType());
@Override public void readFields(DataInput in) throws IOException { columnSpec = new TableSpec.ColumnSpec(); columnSpec.readFields(in); storeDataType = DataTypes.valueOf(in.readByte()); if (DataTypes.isDecimal(storeDataType)) { DecimalType decimalType = (DecimalType) storeDataType; decimalType.setPrecision(columnSpec.getPrecision()); decimalType.setScale(columnSpec.getScale()); } setDecimal(in.readInt()); setDataTypeSelected(in.readByte()); readMinMax(in); compressorName = in.readUTF(); }
return newDoublePage(meta, doubleData); } else if (!isLVEncoded && storeDataType == DataTypes.BYTE_ARRAY && ( columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE || columnSpec.getColumnType() == ColumnType.PLAIN_VALUE)) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length); return newComplexLVBytesPage(columnSpec, lvVarBytes, CarbonCommonConstants.SHORT_SIZE_IN_BYTE, meta.getCompressorName()); } else if (isLVEncoded && storeDataType == DataTypes.BYTE_ARRAY && columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length); return newFixedByteArrayPage(columnSpec, lvVarBytes, 3, meta.getCompressorName()); } else if (storeDataType == DataTypes.BYTE_ARRAY && columnSpec.getColumnType() == ColumnType.COMPLEX_STRUCT) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length); return newFixedByteArrayPage(columnSpec, lvVarBytes, CarbonCommonConstants.SHORT_SIZE_IN_BYTE, meta.getCompressorName()); } else if (storeDataType == DataTypes.BYTE_ARRAY && columnSpec.getColumnType() == ColumnType.COMPLEX_ARRAY) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length); return newFixedByteArrayPage(columnSpec, lvVarBytes, CarbonCommonConstants.LONG_SIZE_IN_BYTE, meta.getCompressorName()); } else if (storeDataType == DataTypes.BYTE_ARRAY && columnSpec.getColumnType() == ColumnType.PLAIN_LONG_VALUE) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length); return newLVBytesPage(columnSpec, lvVarBytes,
TableSpec.ColumnSpec.newInstanceLegacy("legacy", stats.getDataType(), ColumnType.MEASURE); DataType dataType = DataType.getDataType(metadata.getType()); if (dataType == DataTypes.BYTE ||
this.columnName = encodedColumnPage.getActualPage().getColumnSpec().getFieldName(); return;
TableSpec.ColumnSpec.newInstance(columnName, DataTypes.BYTE_ARRAY, columnType); ColumnPage dictionaryColumnPage = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor),
if (columnPageEncoderMeta.getColumnSpec().getSchemaDataType() == DataTypes.BOOLEAN) { value = BooleanConvert.boolean2Byte((Boolean) value);
private byte[] getChunkDataInBytes(int rowId) { ColumnType columnType = columnPage.getColumnSpec().getColumnType(); DataType srcDataType = columnPage.getColumnSpec().getSchemaDataType(); DataType targetDataType = columnPage.getDataType(); if (null != localDictionary) {
private void readMinMax(DataInput in) throws IOException { DataType dataType = columnSpec.getSchemaDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { this.setMaxValue(in.readByte());
} else if (i < noDictionaryCount) { if (DataTypeUtil .isPrimitiveColumn(noDictDimensionPages[i].getColumnSpec().getSchemaDataType())) {
String compressorName = meta.getCompressorName(); TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), if (isUnsafeEnabled(meta)) { page = new UnsafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, columnSpec.getSchemaDataType(), compressorName), rowId); } else { page = new SafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, columnSpec.getSchemaDataType(), compressorName), rowId);
switch (columnSpec.getColumnType()) { case COMPLEX_ARRAY: case COMPLEX_STRUCT:
return compressor.compressByte(getDecimalPage()); } else if (dataType == DataTypes.BYTE_ARRAY && columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.COMPLEX_PRIMITIVE) { return compressor.compressByte(getComplexChildrenLVFlattenedBytePage()); } else if (dataType == DataTypes.BYTE_ARRAY && (columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.COMPLEX_STRUCT || columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.COMPLEX_ARRAY || columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.PLAIN_LONG_VALUE || columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.PLAIN_VALUE)) { return compressor.compressByte(getComplexParentFlattenedBytePage()); } else if (dataType == DataTypes.BYTE_ARRAY) {
private Object getValue(ColumnPage page, int rowId) { DataType type = page.getColumnSpec().getSchemaDataType(); Object value = null; if (type == DataTypes.BYTE) {
return getDecimalPage().length; } else if (dataType == DataTypes.BYTE_ARRAY && columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.COMPLEX_PRIMITIVE) { return getComplexChildrenLVFlattenedBytePage().length; } else if (dataType == DataTypes.BYTE_ARRAY && (columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.COMPLEX_STRUCT || columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.COMPLEX_ARRAY || columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.PLAIN_LONG_VALUE || columnPageEncoderMeta.getColumnSpec().getColumnType() == ColumnType.PLAIN_VALUE)) { return getComplexParentFlattenedBytePage().length; } else if (dataType == DataTypes.BYTE_ARRAY) {
isInvertedIndex(columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE, columnSpec)); } else { isInvertedIndex(columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE, columnSpec));
if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { byte value = getByte(rowId); if (columnPageEncoderMeta.getColumnSpec().getSchemaDataType() == DataTypes.BOOLEAN) { return BooleanConvert.byte2Boolean(value);
/** * method to add complex column data * @param depth * complexColumnIndex of column * @param dataList * dataList */ public void putComplexData(int depth, List<byte[]> dataList) { assert (depth <= this.complexColumnIndex); int positionNumber = currentRowIdList[depth]; for (byte[] value : dataList) { if (columnPages[depth].getDataType() != DataTypes.BYTE_ARRAY) { if ((value == null) || (value.length == 0)) { columnPages[depth].putNull(positionNumber); columnPages[depth].statsCollector.updateNull(positionNumber); columnPages[depth].nullBitSet.set(positionNumber); } else { columnPages[depth].putData(positionNumber, DataTypeUtil .getDataBasedOnDataTypeForNoDictionaryColumn(value, columnPages[depth].getColumnSpec().getSchemaDataType(), false)); } } else { columnPages[depth].putData(positionNumber, value); } positionNumber++; } currentRowIdList[depth] = positionNumber; }
DecimalConverterFactory.DecimalConverter decimalConverter = DecimalConverterFactory.INSTANCE .getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale()); vectorInfo.decimalConverter = decimalConverter; if (DataTypes.isDecimal(meta.getStoreDataType())) {