/** * Below method will be used to convert the thrift presence meta to wrapper * presence meta * * @return wrapper presence meta */ public static BitSet getNullBitSet( org.apache.carbondata.format.PresenceMeta presentMetadataThrift, Compressor compressor) { final byte[] present_bit_stream = presentMetadataThrift.getPresent_bit_stream(); if (null != present_bit_stream) { return BitSet.valueOf(compressor.unCompressByte(present_bit_stream)); } else { return new BitSet(1); } }
/** * Read column schema from binary * * @param schemaArray * @throws IOException */ public static List<ColumnSchema> readColumnSchema(byte[] schemaArray) throws IOException { // uncompress it. schemaArray = CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor().unCompressByte( schemaArray); ByteArrayInputStream schemaStream = new ByteArrayInputStream(schemaArray); DataInput schemaInput = new DataInputStream(schemaStream); List<ColumnSchema> columnSchemas = new ArrayList<>(); int size = schemaInput.readShort(); for (int i = 0; i < size; i++) { ColumnSchema columnSchema = new ColumnSchema(); columnSchema.readFields(schemaInput); columnSchemas.add(columnSchema); } return columnSchemas; }
DataType storeDataType = meta.getStoreDataType(); if (storeDataType == DataTypes.BOOLEAN || storeDataType == DataTypes.BYTE) { byte[] byteData = compressor.unCompressByte(compressedData, offset, length); return newBytePage(meta, byteData); } else if (storeDataType == DataTypes.SHORT) { return newShortPage(meta, shortData); } else if (storeDataType == DataTypes.SHORT_INT) { byte[] shortIntData = compressor.unCompressByte(compressedData, offset, length); return newShortIntPage(meta, shortIntData); } else if (storeDataType == DataTypes.INT) { columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE || columnSpec.getColumnType() == ColumnType.PLAIN_VALUE)) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length); return newComplexLVBytesPage(columnSpec, lvVarBytes, CarbonCommonConstants.SHORT_SIZE_IN_BYTE, meta.getCompressorName()); } else if (isLVEncoded && storeDataType == DataTypes.BYTE_ARRAY && columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length); return newFixedByteArrayPage(columnSpec, lvVarBytes, 3, meta.getCompressorName()); } else if (storeDataType == DataTypes.BYTE_ARRAY && columnSpec.getColumnType() == ColumnType.COMPLEX_STRUCT) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length); return newFixedByteArrayPage(columnSpec, lvVarBytes, CarbonCommonConstants.SHORT_SIZE_IN_BYTE, meta.getCompressorName()); } else if (storeDataType == DataTypes.BYTE_ARRAY && columnSpec.getColumnType() == ColumnType.COMPLEX_ARRAY) { byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length);
DataType storeDataType = meta.getStoreDataType(); if (storeDataType == DataTypes.BYTE) { byte[] byteData = compressor.unCompressByte(compressedData, offset, length); decimalPage = createDecimalPage(meta, byteData.length); decimalPage.setBytePage(byteData); return decimalPage; } else if (storeDataType == DataTypes.SHORT_INT) { byte[] shortIntData = compressor.unCompressByte(compressedData, offset, length); decimalPage = createDecimalPage(meta, shortIntData.length / 3); decimalPage.setShortIntPage(shortIntData); return decimalPage; } else { byte[] lvEncodedBytes = compressor.unCompressByte(compressedData, offset, length); return newDecimalPage(meta, lvEncodedBytes);
ColumnPage decode = decoder.decode(localDictionaryChunk.getDictionary_data(), 0, localDictionaryChunk.getDictionary_data().length); BitSet usedDictionary = BitSet.valueOf(compressor.unCompressByte( localDictionaryChunk.getDictionary_values())); int length = usedDictionary.length();
@Override public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); byte[] unCompressData; if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { int uncompressedLength = compressor.unCompressedLength(input, offset, length); unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); }
@Override public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); byte[] unCompressData; if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { int uncompressedLength = compressor.unCompressedLength(input, offset, length); unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); }
compressor.rawUncompress(pageData.array(), offset, pageMetadata.data_page_length, dataPage); } else { dataPage = compressor.unCompressByte(pageData.array(), offset, pageMetadata.data_page_length); uncompressedSize = dataPage.length;
@Override public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); byte[] unCompressData; if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { int uncompressedLength = compressor.unCompressedLength(input, offset, length); unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); } if (DataTypes.isDecimal(meta.getSchemaDataType())) { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = DecimalConverterFactory.INSTANCE .getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale()); vectorInfo.decimalConverter = decimalConverter; } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); }
@Override public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); byte[] unCompressData; if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { int uncompressedLength = compressor.unCompressedLength(input, offset, length); unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); } if (DataTypes.isDecimal(meta.getSchemaDataType())) { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = DecimalConverterFactory.INSTANCE .getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale()); vectorInfo.decimalConverter = decimalConverter; } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); }
.unCompressByte(encodedColumnPage.getEncodedData().array(), offset, encodedColumnPage.getPageMetadata().data_page_length);
compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); uncompressedLength = unCompressData.length;