DataType type = page.getColumnSpec().getSchemaDataType(); Object value = null; if (type == DataTypes.BYTE) { value = page.getByte(rowId); } else if (type == DataTypes.SHORT) { value = page.getShort(rowId); } else if (type == DataTypes.INT) { value = page.getInt(rowId); } else if (type == DataTypes.LONG) { value = page.getLong(rowId); } else if (type == DataTypes.FLOAT) { value = page.getFloat(rowId); } else if (type == DataTypes.DOUBLE) { value = page.getDouble(rowId); } else if (type == DataTypes.STRING) { byte[] bytes = page.getBytes(rowId); try { value = new String(bytes, 2, bytes.length - 2, "UTF-8"); throw new RuntimeException("unsupported data type " + type); } else if (type == DataTypes.BOOLEAN) { value = page.getBoolean(rowId); } else { LOGGER.error("unsupport data type " + type);
/** * free memory as needed */ public void freeMemory() { if (null != rowOffset) { rowOffset.freeMemory(); rowOffset = null; } }
@Override public long getLong(int rowId) { DataType dataType = columnPage.getDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { return converter.decodeLong(columnPage.getByte(rowId)); } else if (dataType == DataTypes.SHORT) { return converter.decodeLong(columnPage.getShort(rowId)); } else if (dataType == DataTypes.SHORT_INT) { return converter.decodeLong(columnPage.getShortInt(rowId)); } else if (dataType == DataTypes.INT) { return converter.decodeLong(columnPage.getInt(rowId)); } else if (dataType == DataTypes.LONG) { return columnPage.getLong(rowId); } else { throw new RuntimeException("internal error: " + this.toString()); } }
@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new RLEEncoderMeta(inputPage.getColumnSpec(), inputPage.getDataType(), inputPage.getPageSize(), inputPage.getStatistics(), inputPage.getColumnCompressorName()); }
@Override public byte[] getComplexParentFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength]; for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { int length = (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId)); copyBytes(rowId, data, offset, length); offset += length; } return data; }
@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, inputPage.getStatistics(), inputPage.getColumnCompressorName()); }
Object[] data = new Object[input.getActualRowCount()]; if (srcDataType == DataTypes.BYTE || srcDataType == DataTypes.BOOLEAN) { for (int i = 0; i < input.getActualRowCount(); i++) { data[i] = input.getByte(i); for (int i = 0; i < input.getActualRowCount(); i++) { data[i] = input.getShort(i); for (int i = 0; i < input.getActualRowCount(); i++) { data[i] = input.getShortInt(i); for (int i = 0; i < input.getActualRowCount(); i++) { data[i] = input.getInt(i); for (int i = 0; i < input.getActualRowCount(); i++) { data[i] = input.getLong(i); for (int i = 0; i < input.getActualRowCount(); i++) { data[i] = input.getFloat(i); for (int i = 0; i < input.getActualRowCount(); i++) { data[i] = input.getDouble(i);
@Override public double getDouble(int rowId) { DataType dataType = columnPage.getDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { return converter.decodeDouble(columnPage.getByte(rowId)); } else if (dataType == DataTypes.SHORT) { return converter.decodeDouble(columnPage.getShort(rowId)); } else if (dataType == DataTypes.SHORT_INT) { return converter.decodeDouble(columnPage.getShortInt(rowId)); } else if (dataType == DataTypes.INT) { return converter.decodeDouble(columnPage.getInt(rowId)); } else if (dataType == DataTypes.LONG) { return converter.decodeDouble(columnPage.getLong(rowId)); } else if (dataType == DataTypes.FLOAT) { return converter.decodeDouble(columnPage.getFloat(rowId)); } else if (dataType == DataTypes.DOUBLE) { return columnPage.getDouble(rowId); } else { throw new RuntimeException("internal error: " + this.toString()); } }
public static Object getMeasureObjectBasedOnDataType(ColumnPage measurePage, int index, DataType dataType, CarbonMeasure carbonMeasure) { if (dataType == DataTypes.BOOLEAN) { return measurePage.getBoolean(index); } else if (dataType == DataTypes.SHORT) { return (short) measurePage.getLong(index); } else if (dataType == DataTypes.INT) { return (int) measurePage.getLong(index); } else if (dataType == DataTypes.LONG) { return measurePage.getLong(index); } else if (dataType == DataTypes.FLOAT) { return measurePage.getFloat(index); } else if (dataType == DataTypes.BYTE) { return measurePage.getByte(index); } else if (DataTypes.isDecimal(dataType)) { BigDecimal bigDecimalMsrValue = measurePage.getDecimal(index); if (null != bigDecimalMsrValue && carbonMeasure.getScale() > bigDecimalMsrValue.scale()) { bigDecimalMsrValue = bigDecimalMsrValue.setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); } if (null != bigDecimalMsrValue) { return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision()); } else { return null; } } else { return measurePage.getDouble(index); } }
return getNull(rowId); byte value = getByte(rowId); if (columnPageEncoderMeta.getColumnSpec().getSchemaDataType() == DataTypes.BOOLEAN) { return BooleanConvert.byte2Boolean(value); return getShort(rowId); } else if (dataType == DataTypes.INT) { return getInt(rowId); } else if (dataType == DataTypes.LONG) { return getLong(rowId); } else if (dataType == DataTypes.DOUBLE) { return getDouble(rowId); } else if (DataTypes.isDecimal(dataType)) { return getDecimal(rowId); } else if (dataType == DataTypes.STRING || dataType == DataTypes.BYTE_ARRAY || dataType == DataTypes.VARCHAR) { return getBytes(rowId); } else { throw new RuntimeException("unsupported data type: " + dataType);
private byte[] getChunkDataInBytes(int rowId) { ColumnType columnType = columnPage.getColumnSpec().getColumnType(); DataType srcDataType = columnPage.getColumnSpec().getSchemaDataType(); DataType targetDataType = columnPage.getDataType(); if (null != localDictionary) { return localDictionary .getDictionaryValue(CarbonUtil.getSurrogateInternal(columnPage.getBytes(rowId), 0, 3)); } else if ((columnType == ColumnType.COMPLEX_PRIMITIVE && isAdaptiveEncoded()) || ( columnType == ColumnType.PLAIN_VALUE && DataTypeUtil.isPrimitiveColumn(srcDataType))) { if (srcDataType == DataTypes.FLOAT) { float floatData = columnPage.getFloat(rowId); return ByteUtil.toXorBytes(floatData); } else if (srcDataType == DataTypes.DOUBLE) { double doubleData = columnPage.getDouble(rowId); return ByteUtil.toXorBytes(doubleData); } else if (DataTypes.isDecimal(srcDataType)) { == DataTypes.INT) || (srcDataType == DataTypes.LONG) || (srcDataType == DataTypes.TIMESTAMP)) { long longData = columnPage.getLong(rowId); if ((srcDataType == DataTypes.BYTE)) { byte out = (byte) longData; return columnPage.getBytes(rowId); } else { throw new RuntimeException("unsupported type: " + targetDataType); if ((srcDataType == DataTypes.BYTE) || (srcDataType == DataTypes.BOOLEAN)) { byte[] out = new byte[1]; out[0] = (columnPage.getByte(rowId));
/** * Get null at rowId */ private Object getNull(int rowId) { Object result; DataType dataType = columnPageEncoderMeta.getStoreDataType(); if (dataType == DataTypes.BOOLEAN) { result = getBoolean(rowId); } else if (dataType == DataTypes.BYTE) { result = getByte(rowId); if (columnPageEncoderMeta.getColumnSpec().getSchemaDataType() == DataTypes.BOOLEAN) { result = BooleanConvert.byte2Boolean((byte)result); } } else if (dataType == DataTypes.SHORT) { result = getShort(rowId); } else if (dataType == DataTypes.INT) { result = getInt(rowId); } else if (dataType == DataTypes.LONG) { result = getLong(rowId); } else if (dataType == DataTypes.DOUBLE) { result = getDouble(rowId); } else if (DataTypes.isDecimal(dataType)) { result = getDecimal(rowId); } else { throw new IllegalArgumentException("unsupported data type: " + dataType); } return result; }
@Override public FallbackEncodedColumnPage call() throws Exception { int pageSize = encodedColumnPage.getActualPage().getPageSize(); int offset = 0; int[] reverseInvertedIndex = new int[pageSize]; encodedColumnPage.getActualPage().getColumnPageEncoderMeta().getCompressorName()) .unCompressByte(encodedColumnPage.getEncodedData().array(), offset, encodedColumnPage.getPageMetadata().data_page_length); encodedColumnPage.getActualPage().disableLocalDictEncoding(); ColumnPage.newPage(encodedColumnPage.getActualPage().getColumnPageEncoderMeta(), encodedColumnPage.getActualPage().getPageSize()); .getKeyGenerator(new int[] { CarbonCommonConstants.LOCAL_DICTIONARY_MAX + 1 }); actualDataColumnPage.setStatsCollector(encodedColumnPage.getActualPage().statsCollector); int keyArray = (int) keyGenerator.getKeyArray(bytes, index)[0]; actualDataColumnPage .putBytes(rowId++, localDictionaryGenerator.getDictionaryKeyBasedOnValue(keyArray)); TableSpec.ColumnSpec columnSpec = encodedColumnPage.getActualPage().getColumnSpec(); FallbackEncodedColumnPage fallBackEncodedColumnPage = CarbonUtil.getFallBackEncodedColumnPage(actualDataColumnPage, pageIndex, columnSpec); actualDataColumnPage.freeMemory(); return fallBackEncodedColumnPage;
/** * Create a new column page with input data type and page size. */ protected LocalDictColumnPage(ColumnPage actualDataColumnPage, ColumnPage encodedColumnpage, LocalDictionaryGenerator localDictionaryGenerator, boolean isComplexTypePrimitive, boolean isDecoderBasedFallBackEnabled) { super(actualDataColumnPage.getColumnPageEncoderMeta(), actualDataColumnPage.getPageSize()); // if threshold is not reached then create page level dictionary // for encoding with local dictionary if (!localDictionaryGenerator.isThresholdReached()) { pageLevelDictionary = new PageLevelDictionary(localDictionaryGenerator, actualDataColumnPage.getColumnSpec().getFieldName(), actualDataColumnPage.getDataType(), isComplexTypePrimitive, actualDataColumnPage.getColumnCompressorName()); this.encodedDataColumnPage = encodedColumnpage; this.keyGenerator = KeyGeneratorFactory .getKeyGenerator(new int[] { CarbonCommonConstants.LOCAL_DICTIONARY_MAX + 1 }); this.dummyKey = new int[1]; } else { // else free the encoded column page memory as its of no use encodedColumnpage.freeMemory(); } this.isDecoderBasedFallBackEnabled = isDecoderBasedFallBackEnabled; this.actualDataColumnPage = actualDataColumnPage; }
/** * Get the new column page based on the sorted data * * @param input * @return * @throws MemoryException */ public ColumnPage getSortedColumnPageIfRequired(ColumnPage input) throws MemoryException { if (null != indexStorage) { Object[] dataPage = indexStorage.getDataPage(); ColumnPageEncoderMeta columnPageEncoderMeta = new ColumnPageEncoderMeta(input.getColumnSpec(), input.getDataType(), input.getColumnPageEncoderMeta().getCompressorName()); ColumnPage columnPage = ColumnPage.newPage(columnPageEncoderMeta, input.getPageSize()); putDataToPage(columnPage, dataPage); return columnPage; } else { return input; } }
@Override public BigDecimal getDecimal(int rowId) { DecimalConverterFactory.DecimalConverter decimalConverter = ((DecimalColumnPage) columnPage).getDecimalConverter(); DataType dataType = columnPage.getDataType(); if (dataType == DataTypes.BYTE) { return decimalConverter.getDecimal(converter.decodeLong(columnPage.getByte(rowId))); } else if (dataType == DataTypes.SHORT) { return decimalConverter.getDecimal(converter.decodeLong(columnPage.getShort(rowId))); } else if (dataType == DataTypes.SHORT_INT) { return decimalConverter.getDecimal(converter.decodeLong(columnPage.getShortInt(rowId))); } else if (dataType == DataTypes.INT) { return decimalConverter.getDecimal(converter.decodeLong(columnPage.getInt(rowId))); } else if (dataType == DataTypes.LONG || DataTypes.isDecimal(dataType)) { return columnPage.getDecimal(rowId); } else { throw new RuntimeException("internal error: " + this.toString()); } }
public byte[] encodeAndCompressPage(ColumnPage input, ColumnPageValueConverter converter, Compressor compressor) throws MemoryException, IOException { encodedPage = ColumnPage.newPage( new ColumnPageEncoderMeta(input.getColumnPageEncoderMeta().getColumnSpec(), targetDataType, input.getColumnPageEncoderMeta().getCompressorName()), input.getPageSize()); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForNoDictionary(getPageBasedOnDataType(input), input.getDataType(), isInvertedIndex); } ColumnPage columnPage = getSortedColumnPageIfRequired(input); columnPage.convertValue(converter); byte[] result = encodedPage.compress(compressor); return result; }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { vector.putLong(vectorOffset, dataChunk.getLong(i)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { vector.putLong(vectorOffset, dataChunk.getLong(i)); } vectorOffset++; } } }
/** * method to add complex column data * @param depth * complexColumnIndex of column * @param dataList * dataList */ public void putComplexData(int depth, List<byte[]> dataList) { assert (depth <= this.complexColumnIndex); int positionNumber = currentRowIdList[depth]; for (byte[] value : dataList) { if (columnPages[depth].getDataType() != DataTypes.BYTE_ARRAY) { if ((value == null) || (value.length == 0)) { columnPages[depth].putNull(positionNumber); columnPages[depth].statsCollector.updateNull(positionNumber); columnPages[depth].nullBitSet.set(positionNumber); } else { columnPages[depth].putData(positionNumber, DataTypeUtil .getDataBasedOnDataTypeForNoDictionaryColumn(value, columnPages[depth].getColumnSpec().getSchemaDataType(), false)); } } else { columnPages[depth].putData(positionNumber, value); } positionNumber++; } currentRowIdList[depth] = positionNumber; }
.getColumnIndex()].decodeColumnPage(pageIndex); if (msrType == DataTypes.BOOLEAN) { msrValue = columnPage.getBoolean(index); } else if (msrType == DataTypes.SHORT) { msrValue = (short) columnPage.getLong(index); } else if (msrType == DataTypes.INT) { msrValue = (int) columnPage.getLong(index); } else if (msrType == DataTypes.LONG) { msrValue = columnPage.getLong(index); } else if (DataTypes.isDecimal(msrType)) { BigDecimal bigDecimalValue = columnPage.getDecimal(index); if (null != bigDecimalValue && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale() msrValue = columnPage.getDouble(index); columnPage.getNullBits().get(index) ? null : msrValue;