public CarbonDictionary getLocalDictionary() { if (null != getDataChunkV3() && null != getDataChunkV3().local_dictionary && null == localDictionary) { try { String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta( getDataChunkV3().data_chunk_list.get(0).chunk_meta); Compressor compressor = CompressorFactory.getInstance().getCompressor(compressorName); localDictionary = getDictionary(getDataChunkV3().local_dictionary, compressor); } catch (IOException | MemoryException e) { throw new RuntimeException(e); } } return localDictionary; }
private ColumnPage decodeDimensionByMeta(DataChunk2 pageMetadata, ByteBuffer pageData, int offset, boolean isLocalDictEncodedPage, ColumnVectorInfo vectorInfo, BitSet nullBitSet, ReusableDataBuffer reusableDataBuffer) throws IOException, MemoryException { List<Encoding> encodings = pageMetadata.getEncoders(); List<ByteBuffer> encoderMetas = pageMetadata.getEncoder_meta(); String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta( pageMetadata.getChunk_meta()); ColumnPageDecoder decoder = encodingFactory.createDecoder(encodings, encoderMetas, compressorName, vectorInfo != null); if (vectorInfo != null) { decoder .decodeAndFillVector(pageData.array(), offset, pageMetadata.data_page_length, vectorInfo, nullBitSet, isLocalDictEncodedPage, pageMetadata.numberOfRowsInpage, reusableDataBuffer); return null; } else { return decoder .decode(pageData.array(), offset, pageMetadata.data_page_length, isLocalDictEncodedPage); } }
/** * Decode measure column page with page header and raw data starting from offset */ protected ColumnPage decodeMeasure(DataChunk2 pageMetadata, ByteBuffer pageData, int offset, ColumnVectorInfo vectorInfo, BitSet nullBitSet, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { List<Encoding> encodings = pageMetadata.getEncoders(); org.apache.carbondata.core.metadata.encoder.Encoding.validateEncodingTypes(encodings); List<ByteBuffer> encoderMetas = pageMetadata.getEncoder_meta(); String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta(pageMetadata.getChunk_meta()); ColumnPageDecoder codec = encodingFactory.createDecoder(encodings, encoderMetas, compressorName, vectorInfo != null); if (vectorInfo != null) { codec.decodeAndFillVector(pageData.array(), offset, pageMetadata.data_page_length, vectorInfo, nullBitSet, false, pageMetadata.numberOfRowsInpage, reusableDataBuffer); return null; } else { return codec .decode(pageData.array(), offset, pageMetadata.data_page_length); } } }
private DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk rawColumnPage, int pageNumber, ColumnVectorInfo vectorInfo, ReusableDataBuffer reusableDataBuffer) throws IOException, MemoryException { // data chunk of blocklet column DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3(); // get the data buffer ByteBuffer rawData = rawColumnPage.getRawData(); DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber); String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta( pageMetadata.getChunk_meta()); this.compressor = CompressorFactory.getInstance().getCompressor(compressorName); // calculating the start point of data // as buffer can contain multiple column data, start point will be datachunkoffset + // data chunk length + page offset int offset = (int) rawColumnPage.getOffSet() + dimensionChunksLength .get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber); // first read the data and uncompressed it return decodeDimension(rawColumnPage, rawData, pageMetadata, offset, vectorInfo, reusableDataBuffer); }
private ColumnPage decodeColumnPage(MeasureRawColumnChunk rawColumnChunk, int pageNumber, ColumnVectorInfo vectorInfo, ReusableDataBuffer reusableDataBuffer) throws IOException, MemoryException { // data chunk of blocklet column DataChunk3 dataChunk3 = rawColumnChunk.getDataChunkV3(); // data chunk of page DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber); String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta( pageMetadata.getChunk_meta()); this.compressor = CompressorFactory.getInstance().getCompressor(compressorName); // calculating the start point of data // as buffer can contain multiple column data, start point will be datachunkoffset + // data chunk length + page offset int offset = (int) rawColumnChunk.getOffSet() + measureColumnChunkLength.get(rawColumnChunk.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber); BitSet nullBitSet = QueryUtil.getNullBitSet(pageMetadata.presence, this.compressor); ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnChunk.getRawData(), offset, vectorInfo, nullBitSet, reusableDataBuffer); if (decodedPage == null) { return null; } decodedPage.setNullBits(nullBitSet); return decodedPage; }
/** * Below method will be used to convert the compressed measure chunk raw data to actual data * * @param rawColumnPage measure raw chunk * @param pageNumber number * @return DimensionColumnDataChunk */ @Override public ColumnPage decodeColumnPage( MeasureRawColumnChunk rawColumnPage, int pageNumber, ReusableDataBuffer reusableDataBuffer) throws IOException, MemoryException { // data chunk of blocklet column DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3(); // data chunk of page DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber); String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta( pageMetadata.getChunk_meta()); this.compressor = CompressorFactory.getInstance().getCompressor(compressorName); // calculating the start point of data // as buffer can contain multiple column data, start point will be datachunkoffset + // data chunk length + page offset long offset = rawColumnPage.getOffSet() + measureColumnChunkLength .get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber); ByteBuffer buffer = rawColumnPage.getFileReader() .readByteBuffer(filePath, offset, pageMetadata.data_page_length); BitSet nullBitSet = QueryUtil.getNullBitSet(pageMetadata.presence, this.compressor); ColumnPage decodedPage = decodeMeasure(pageMetadata, buffer, 0, null, nullBitSet, reusableDataBuffer); decodedPage.setNullBits(nullBitSet); return decodedPage; }
CarbonMetadataUtil.getCompressorNameFromChunkMeta(pageMetadata.getChunk_meta()));