@Override protected byte[] compress(byte[] in) { if (in == null) { throw new NullPointerException("Can't compress null"); } LZ4Compressor compressor = lz4Factory.fastCompressor(); byte[] out = new byte[compressor.maxCompressedLength(in.length)]; int compressedLength = compressor.compress(in, 0, in.length, out, 0); getLogger().debug("Compressed %d bytes to %d", in.length, compressedLength); return ByteBuffer.allocate(Integer.BYTES + compressedLength) .putInt(in.length) .put(out, 0, compressedLength) .array(); }
@Override public ByteBuf encode(Object in) throws IOException { ByteBuf bytes = null; try { LZ4Compressor compressor = factory.fastCompressor(); bytes = innerCodec.getValueEncoder().encode(in); ByteBuffer srcBuf = bytes.internalNioBuffer(bytes.readerIndex(), bytes.readableBytes()); int outMaxLength = compressor.maxCompressedLength(bytes.readableBytes()); ByteBuf out = ByteBufAllocator.DEFAULT.buffer(outMaxLength + DECOMPRESSION_HEADER_SIZE); out.writeInt(bytes.readableBytes()); ByteBuffer outBuf = out.internalNioBuffer(out.writerIndex(), out.writableBytes()); int pos = outBuf.position(); compressor.compress(srcBuf, outBuf); int compressedLength = outBuf.position() - pos; out.writerIndex(out.writerIndex() + compressedLength); return out; } finally { if (bytes != null) { bytes.release(); } } } };
@Override public ByteBuf encode(Object in) throws IOException { ByteBuf bytes = null; try { LZ4Compressor compressor = factory.fastCompressor(); bytes = innerCodec.getValueEncoder().encode(in); ByteBuffer srcBuf = bytes.internalNioBuffer(bytes.readerIndex(), bytes.readableBytes()); int outMaxLength = compressor.maxCompressedLength(bytes.readableBytes()); ByteBuf out = ByteBufAllocator.DEFAULT.buffer(outMaxLength + DECOMPRESSION_HEADER_SIZE); out.writeInt(bytes.readableBytes()); ByteBuffer outBuf = out.internalNioBuffer(out.writerIndex(), out.writableBytes()); int pos = outBuf.position(); compressor.compress(srcBuf, outBuf); int compressedLength = outBuf.position() - pos; out.writerIndex(out.writerIndex() + compressedLength); return out; } finally { if (bytes != null) { bytes.release(); } } } };
/** * Creates a new customizable LZ4 encoder. * * @param factory user customizable {@link LZ4Factory} instance * which may be JNI bindings to the original C implementation, a pure Java implementation * or a Java implementation that uses the {@link sun.misc.Unsafe} * @param highCompressor if {@code true} codec will use compressor which requires more memory * and is slower but compresses more efficiently * @param blockSize the maximum number of bytes to try to compress at once, * must be >= 64 and <= 32 M * @param checksum the {@link Checksum} instance to use to check data for integrity * @param maxEncodeSize the maximum size for an encode (compressed) buffer */ public Lz4FrameEncoder(LZ4Factory factory, boolean highCompressor, int blockSize, Checksum checksum, int maxEncodeSize) { if (factory == null) { throw new NullPointerException("factory"); } if (checksum == null) { throw new NullPointerException("checksum"); } compressor = highCompressor ? factory.highCompressor() : factory.fastCompressor(); this.checksum = ByteBufChecksum.wrapChecksum(checksum); compressionLevel = compressionLevel(blockSize); this.blockSize = blockSize; this.maxEncodeSize = ObjectUtil.checkPositive(maxEncodeSize, "maxEncodeSize"); finished = false; }
/** * Creates a new customizable LZ4 encoder. * * @param factory user customizable {@link LZ4Factory} instance * which may be JNI bindings to the original C implementation, a pure Java implementation * or a Java implementation that uses the {@link sun.misc.Unsafe} * @param highCompressor if {@code true} codec will use compressor which requires more memory * and is slower but compresses more efficiently * @param blockSize the maximum number of bytes to try to compress at once, * must be >= 64 and <= 32 M * @param checksum the {@link Checksum} instance to use to check data for integrity * @param maxEncodeSize the maximum size for an encode (compressed) buffer */ public Lz4FrameEncoder(LZ4Factory factory, boolean highCompressor, int blockSize, Checksum checksum, int maxEncodeSize) { if (factory == null) { throw new NullPointerException("factory"); } if (checksum == null) { throw new NullPointerException("checksum"); } compressor = highCompressor ? factory.highCompressor() : factory.fastCompressor(); this.checksum = ByteBufChecksum.wrapChecksum(checksum); compressionLevel = compressionLevel(blockSize); this.blockSize = blockSize; this.maxEncodeSize = ObjectUtil.checkPositive(maxEncodeSize, "maxEncodeSize"); finished = false; }
/** * Creates a new customizable LZ4 encoder. * * @param factory user customizable {@link LZ4Factory} instance * which may be JNI bindings to the original C implementation, a pure Java implementation * or a Java implementation that uses the {@link sun.misc.Unsafe} * @param highCompressor if {@code true} codec will use compressor which requires more memory * and is slower but compresses more efficiently * @param blockSize the maximum number of bytes to try to compress at once, * must be >= 64 and <= 32 M * @param checksum the {@link Checksum} instance to use to check data for integrity * @param maxEncodeSize the maximum size for an encode (compressed) buffer */ public Lz4FrameEncoder(LZ4Factory factory, boolean highCompressor, int blockSize, Checksum checksum, int maxEncodeSize) { if (factory == null) { throw new NullPointerException("factory"); } if (checksum == null) { throw new NullPointerException("checksum"); } compressor = highCompressor ? factory.highCompressor() : factory.fastCompressor(); this.checksum = ByteBufChecksum.wrapChecksum(checksum); compressionLevel = compressionLevel(blockSize); this.blockSize = blockSize; this.maxEncodeSize = ObjectUtil.checkPositive(maxEncodeSize, "maxEncodeSize"); finished = false; }
public LZ4CompressionCodec() { this.compressor = LZ4Factory.fastestInstance().fastCompressor(); this.fastDecompressor = LZ4Factory.fastestInstance().fastDecompressor(); this.safeDecompressor = LZ4Factory.fastestInstance().safeDecompressor(); }
/** * Create a new {@link OutputStream} that will compress data using the LZ4 algorithm. * * @param out The output stream to compress * @param blockSize Default: 4. The block size used during compression. 4=64kb, 5=256kb, 6=1mb, 7=4mb. All other * values will generate an exception * @param blockChecksum Default: false. When true, a XXHash32 checksum is computed and appended to the stream for * every block of data * @param useBrokenFlagDescriptorChecksum Default: false. When true, writes an incorrect FrameDescriptor checksum * compatible with older kafka clients. * @throws IOException */ public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum, boolean useBrokenFlagDescriptorChecksum) throws IOException { this.out = out; compressor = LZ4Factory.fastestInstance().fastCompressor(); checksum = XXHashFactory.fastestInstance().hash32(); this.useBrokenFlagDescriptorChecksum = useBrokenFlagDescriptorChecksum; bd = new BD(blockSize); flg = new FLG(blockChecksum); bufferOffset = 0; maxBlockSize = bd.getBlockMaximumSize(); buffer = new byte[maxBlockSize]; compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)]; finished = false; writeHeader(); }
private LZ4Compressor() { final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); logger.info("Using {}", lz4Factory.toString()); compressor = lz4Factory.fastCompressor(); decompressor = lz4Factory.fastDecompressor(); }
/** * Create a new instance which compresses with the standard LZ4 compression * algorithm. * @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor) * @see LZ4Factory#fastCompressor() */ public LZ4BlockOutputStream(OutputStream out, int blockSize) { this(out, blockSize, LZ4Factory.fastestInstance().fastCompressor()); }
private LZ4Compressor() { final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); compressor = lz4Factory.fastCompressor(); decompressor = lz4Factory.decompressor(); }
public ClickHouseLZ4OutputStream(OutputStream stream, int maxCompressBlockSize) { dataWrapper = new LittleEndianDataOutputStream(stream); compressor = factory.fastCompressor(); currentBlock = new byte[maxCompressBlockSize]; compressedBlock = new byte[compressor.maxCompressedLength(maxCompressBlockSize)]; }
private LZ4Compressor() { final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); compressor = lz4Factory.fastCompressor(); decompressor = lz4Factory.fastDecompressor(); }
public ClickHouseLZ4OutputStream(OutputStream stream, int maxCompressBlockSize) { dataWrapper = new LittleEndianDataOutputStream(stream); compressor = factory.fastCompressor(); currentBlock = new byte[maxCompressBlockSize]; compressedBlock = new byte[compressor.maxCompressedLength(maxCompressBlockSize)]; }
private LZ4Compressor() { final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); compressor = lz4Factory.fastCompressor(); decompressor = lz4Factory.decompressor(); }
private LZ4Compressor() { final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); logger.info("Using {}", lz4Factory.toString()); compressor = lz4Factory.fastCompressor(); decompressor = lz4Factory.fastDecompressor(); }
private LZ4Compressor() { final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); logger.info("Using {}", lz4Factory.toString()); compressor = lz4Factory.fastCompressor(); decompressor = lz4Factory.fastDecompressor(); }
@Override public void write(RevObject o, OutputStream out) throws IOException { final int blockSize = 1 << 16; LZ4Compressor compressor = lz4factory.fastCompressor(); Checksum checksum = newChecksum(); LZ4BlockOutputStream cout = new LZ4BlockOutputStream(out, blockSize, compressor, checksum, true); factory.write(o, cout); cout.finish();// same as close but not closing the wrapped stream }
@Override public void write(RevObject o, OutputStream out) throws IOException { final int blockSize = 1 << 16; LZ4Compressor compressor = lz4factory.fastCompressor(); Checksum checksum = newChecksum(); LZ4BlockOutputStream cout = new LZ4BlockOutputStream(out, blockSize, compressor, checksum, true); factory.write(o, cout); cout.finish();// same as close but not closing the wrapped stream }
@Override public void serialize(final Object o, final OutputStream os) throws Exception { final LZ4BlockOutputStream lz4os = new LZ4BlockOutputStream(os, 32*1024, LZ4Factory.fastestInstance().fastCompressor()); try { getDelegate().serialize(o, lz4os); } finally { lz4os.finish(); } }