protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk, DataChunk2 measureColumnChunk, int copyPoint, ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException { assert (measureColumnChunk.getEncoder_meta().size() > 0); List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta(); byte[] encodedMeta = encoder_meta.get(0).array(); ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV2(encodedMeta); ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta, CompressorFactory.NativeSupportedCompressor.SNAPPY.getName()); byte[] rawData = measureRawColumnChunk.getRawData().array(); return codec .decode(rawData, copyPoint, measureColumnChunk.data_page_length); } }
private CompressorFactory() { for (NativeSupportedCompressor nativeSupportedCompressor : NativeSupportedCompressor.values()) { allSupportedCompressors.put(nativeSupportedCompressor.getName(), nativeSupportedCompressor.getCompressor()); } }
@Override public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber, ReusableDataBuffer reusableDataBuffer) throws IOException, MemoryException { int blockIndex = measureRawColumnChunk.getColumnIndex(); DataChunk dataChunk = measureColumnChunks.get(blockIndex); ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0); ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta, CompressorFactory.NativeSupportedCompressor.SNAPPY.getName()); ColumnPage decodedPage = codec.decode(measureRawColumnChunk.getRawData().array(), (int) measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength()); decodedPage.setNullBits(dataChunk.getNullValueIndexForColumn()); return decodedPage; } }
compressorName = header.getCompressor_name(); } else { compressorName = CompressorFactory.NativeSupportedCompressor.SNAPPY.getName();
private byte[] getSyncMarker(String filePath) throws IOException { CarbonHeaderReader headerReader = new CarbonHeaderReader(filePath); FileHeader header = headerReader.readHeader(); // legacy store does not have this member if (header.isSetCompressor_name()) { compressorName = header.getCompressor_name(); } else { compressorName = CompressorFactory.NativeSupportedCompressor.SNAPPY.getName(); } return header.getSync_marker(); }
/** * get the compressor name from chunk meta * before 1.5.0, we only support snappy and do not have compressor_name field; * after 1.5.0, we directly get the compressor from the compressor_name field */ public static String getCompressorNameFromChunkMeta(ChunkCompressionMeta chunkCompressionMeta) { if (chunkCompressionMeta.isSetCompressor_name()) { return chunkCompressionMeta.getCompressor_name(); } else { // this is for legacy store before 1.5.0 return CompressorFactory.NativeSupportedCompressor.SNAPPY.getName(); } } /**