public StreamBlockletReader(byte[] syncMarker, InputStream in, long limit, boolean isHeaderPresent, String compressorName) { this.syncMarker = syncMarker; syncLen = syncMarker.length; syncBuffer = new byte[syncLen]; this.in = in; limitStart = limit; limitEnd = limitStart + syncLen; this.isHeaderPresent = isHeaderPresent; this.compressor = CompressorFactory.getInstance().getCompressor(compressorName); }
/** * get the default compressor. * This method can only be called in data load procedure to compress column page. * In query procedure, we should read the compressor information from the metadata * in datafiles when we want to decompress the content. */ public Compressor getCompressor() { String compressorType = CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR); return getCompressor(compressorType); }
StreamBlockletWriter(int maxSize, int maxRowNum, int rowSize, int dimCountWithoutComplex, int measureCount, DataType[] measureDataTypes, String compressorName) { buffer = new byte[maxSize]; this.maxSize = maxSize; this.maxRowNum = maxRowNum; this.rowSize = rowSize; this.dimCountWithoutComplex = dimCountWithoutComplex; this.measureCount = measureCount; this.measureDataTypes = measureDataTypes; this.compressor = CompressorFactory.getInstance().getCompressor(compressorName); initializeStatsCollector(); }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); return input.compress(compressor); }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
private void validateAndSetColumnCompressor(CarbonLoadModel carbonLoadModel) throws InvalidLoadOptionException { try { String columnCompressor = carbonLoadModel.getColumnCompressor(); if (StringUtils.isBlank(columnCompressor)) { columnCompressor = CarbonProperties.getInstance().getProperty( CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR); } // check and load compressor CompressorFactory.getInstance().getCompressor(columnCompressor); carbonLoadModel.setColumnCompressor(columnCompressor); } catch (Exception e) { LOGGER.error(e); throw new InvalidLoadOptionException("Failed to load the compressor"); } }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
public CarbonDictionary getLocalDictionary() { if (null != getDataChunkV3() && null != getDataChunkV3().local_dictionary && null == localDictionary) { try { String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta( getDataChunkV3().data_chunk_list.get(0).chunk_meta); Compressor compressor = CompressorFactory.getInstance().getCompressor(compressorName); localDictionary = getDictionary(getDataChunkV3().local_dictionary, compressor); } catch (IOException | MemoryException e) { throw new RuntimeException(e); } } return localDictionary; }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
private DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk rawColumnPage, int pageNumber, ColumnVectorInfo vectorInfo, ReusableDataBuffer reusableDataBuffer) throws IOException, MemoryException { // data chunk of blocklet column DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3(); // get the data buffer ByteBuffer rawData = rawColumnPage.getRawData(); DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber); String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta( pageMetadata.getChunk_meta()); this.compressor = CompressorFactory.getInstance().getCompressor(compressorName); // calculating the start point of data // as buffer can contain multiple column data, start point will be datachunkoffset + // data chunk length + page offset int offset = (int) rawColumnPage.getOffSet() + dimensionChunksLength .get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber); // first read the data and uncompressed it return decodeDimension(rawColumnPage, rawData, pageMetadata, offset, vectorInfo, reusableDataBuffer); }
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage = new BlockIndexerStorageForShort(inputPage.getByteArrayPage(), false, false, false); byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); byte[] compressed = compressor.compressByte(flattened); super.indexStorage = indexStorage; super.compressedDataPage = compressed; }
private void fillNullBitSet(ColumnPage inputPage, DataChunk2 dataChunk) { PresenceMeta presenceMeta = new PresenceMeta(); presenceMeta.setPresent_bit_streamIsSet(true); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); presenceMeta.setPresent_bit_stream( compressor.compressByte(inputPage.getNullBits().toByteArray())); dataChunk.setPresence(presenceMeta); }
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = inputPage.getByteArrayPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, true, false, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, false); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = inputPage.getByteArrayPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, false, false, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, false); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }
@Override protected void encodeIndexStorage(ColumnPage input) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = input.getByteArrayPage(); boolean isDictionary = input.isLocalDictGeneratedPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, isDictionary, !isDictionary, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, isDictionary); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }
@Override public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); byte[] unCompressData; if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { int uncompressedLength = compressor.unCompressedLength(input, offset, length); unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); }
@Override public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); byte[] unCompressData; if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { int uncompressedLength = compressor.unCompressedLength(input, offset, length); unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); }
@Override public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); byte[] unCompressData; if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { int uncompressedLength = compressor.unCompressedLength(input, offset, length); unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); } if (DataTypes.isDecimal(meta.getSchemaDataType())) { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = DecimalConverterFactory.INSTANCE .getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale()); vectorInfo.decimalConverter = decimalConverter; } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); }
@Override public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName()); byte[] unCompressData; if (null != reusableDataBuffer && compressor.supportReusableBuffer()) { int uncompressedLength = compressor.unCompressedLength(input, offset, length); unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength); compressor.rawUncompress(input, offset, length, unCompressData); } else { unCompressData = compressor.unCompressByte(input, offset, length); } if (DataTypes.isDecimal(meta.getSchemaDataType())) { TableSpec.ColumnSpec columnSpec = meta.getColumnSpec(); DecimalConverterFactory.DecimalConverter decimalConverter = DecimalConverterFactory.INSTANCE .getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale()); vectorInfo.decimalConverter = decimalConverter; } converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize); }