@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, stats, inputPage.getColumnCompressorName()); }
@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, stats, inputPage.getColumnCompressorName()); }
@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, stats, inputPage.getColumnCompressorName()); }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); return input.compress(compressor); }
@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, inputPage.getStatistics(), inputPage.getColumnCompressorName()); }
@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), inputPage.getDataType(), inputPage.getStatistics(), inputPage.getColumnCompressorName()); } };
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new RLEEncoderMeta(inputPage.getColumnSpec(), inputPage.getDataType(), inputPage.getPageSize(), inputPage.getStatistics(), inputPage.getColumnCompressorName()); }
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage = new BlockIndexerStorageForShort(inputPage.getByteArrayPage(), false, false, false); byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); byte[] compressed = compressor.compressByte(flattened); super.indexStorage = indexStorage; super.compressedDataPage = compressed; }
/** * Right now it is set to default values. We may use this in future * set the compressor. * before 1.5.0, we set a enum 'compression_codec'; * after 1.5.0, we use string 'compressor_name' instead */ public static ChunkCompressionMeta getChunkCompressorMeta( ColumnPage inputPage, long encodedDataLength) throws IOException { ChunkCompressionMeta chunkCompressionMeta = new ChunkCompressionMeta(); // we will not use this field any longer and will use compressor_name instead, // but in thrift definition, this field is required so we cannot set it to null, otherwise // it will cause deserialization error in runtime (required field cannot be null). chunkCompressionMeta.setCompression_codec(CompressionCodec.DEPRECATED); chunkCompressionMeta.setCompressor_name(inputPage.getColumnCompressorName()); chunkCompressionMeta.setTotal_compressed_size(encodedDataLength); chunkCompressionMeta.setTotal_uncompressed_size(inputPage.getPageLengthInBytes()); return chunkCompressionMeta; }
private void fillNullBitSet(ColumnPage inputPage, DataChunk2 dataChunk) { PresenceMeta presenceMeta = new PresenceMeta(); presenceMeta.setPresent_bit_streamIsSet(true); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); presenceMeta.setPresent_bit_stream( compressor.compressByte(inputPage.getNullBits().toByteArray())); dataChunk.setPresence(presenceMeta); }
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = inputPage.getByteArrayPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, true, false, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, false); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }
@Override void encodeIndexStorage(ColumnPage inputPage) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = inputPage.getByteArrayPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, false, false, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, false); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }
@Override protected void encodeIndexStorage(ColumnPage input) { BlockIndexerStorage<byte[][]> indexStorage; byte[][] data = input.getByteArrayPage(); boolean isDictionary = input.isLocalDictGeneratedPage(); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForShort(data, isDictionary, !isDictionary, isSort); } else { indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, isDictionary); } byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage()); Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); super.compressedDataPage = compressor.compressByte(flattened); super.indexStorage = indexStorage; }
/** * Create a new column page with input data type and page size. */ protected LocalDictColumnPage(ColumnPage actualDataColumnPage, ColumnPage encodedColumnpage, LocalDictionaryGenerator localDictionaryGenerator, boolean isComplexTypePrimitive, boolean isDecoderBasedFallBackEnabled) { super(actualDataColumnPage.getColumnPageEncoderMeta(), actualDataColumnPage.getPageSize()); // if threshold is not reached then create page level dictionary // for encoding with local dictionary if (!localDictionaryGenerator.isThresholdReached()) { pageLevelDictionary = new PageLevelDictionary(localDictionaryGenerator, actualDataColumnPage.getColumnSpec().getFieldName(), actualDataColumnPage.getDataType(), isComplexTypePrimitive, actualDataColumnPage.getColumnCompressorName()); this.encodedDataColumnPage = encodedColumnpage; this.keyGenerator = KeyGeneratorFactory .getKeyGenerator(new int[] { CarbonCommonConstants.LOCAL_DICTIONARY_MAX + 1 }); this.dummyKey = new int[1]; } else { // else free the encoded column page memory as its of no use encodedColumnpage.freeMemory(); } this.isDecoderBasedFallBackEnabled = isDecoderBasedFallBackEnabled; this.actualDataColumnPage = actualDataColumnPage; }