Refine search
@Override protected byte[] compress(byte[] in) { if (in == null) { throw new NullPointerException("Can't compress null"); } LZ4Compressor compressor = lz4Factory.fastCompressor(); byte[] out = new byte[compressor.maxCompressedLength(in.length)]; int compressedLength = compressor.compress(in, 0, in.length, out, 0); getLogger().debug("Compressed %d bytes to %d", in.length, compressedLength); return ByteBuffer.allocate(Integer.BYTES + compressedLength) .putInt(in.length) .put(out, 0, compressedLength) .array(); }
public LZ4CompressionCodec() { this.compressor = LZ4Factory.fastestInstance().fastCompressor(); this.fastDecompressor = LZ4Factory.fastestInstance().fastDecompressor(); this.safeDecompressor = LZ4Factory.fastestInstance().safeDecompressor(); }
/** * Creates a new customizable LZ4 encoder. * * @param factory user customizable {@link LZ4Factory} instance * which may be JNI bindings to the original C implementation, a pure Java implementation * or a Java implementation that uses the {@link sun.misc.Unsafe} * @param highCompressor if {@code true} codec will use compressor which requires more memory * and is slower but compresses more efficiently * @param blockSize the maximum number of bytes to try to compress at once, * must be >= 64 and <= 32 M * @param checksum the {@link Checksum} instance to use to check data for integrity * @param maxEncodeSize the maximum size for an encode (compressed) buffer */ public Lz4FrameEncoder(LZ4Factory factory, boolean highCompressor, int blockSize, Checksum checksum, int maxEncodeSize) { if (factory == null) { throw new NullPointerException("factory"); } if (checksum == null) { throw new NullPointerException("checksum"); } compressor = highCompressor ? factory.highCompressor() : factory.fastCompressor(); this.checksum = ByteBufChecksum.wrapChecksum(checksum); compressionLevel = compressionLevel(blockSize); this.blockSize = blockSize; this.maxEncodeSize = ObjectUtil.checkPositive(maxEncodeSize, "maxEncodeSize"); finished = false; }
/** * Create a new instance which uses the fastest {@link LZ4FastDecompressor} available. * @see LZ4Factory#fastestInstance() * @see #LZ4BlockInputStream(InputStream, LZ4FastDecompressor) */ public LZ4BlockInputStream(InputStream in) { this(in, LZ4Factory.fastestInstance().fastDecompressor()); }
/** * Creates a new LZ4 encoder with hight or fast compression, default block size (64 KB) * and xxhash hashing for Java, based on Yann Collet's work available at * <a href="https://github.com/Cyan4973/xxHash">Github</a>. * * @param highCompressor if {@code true} codec will use compressor which requires more memory * and is slower but compresses more efficiently */ public Lz4FrameEncoder(boolean highCompressor) { this(LZ4Factory.fastestInstance(), highCompressor, DEFAULT_BLOCK_SIZE, XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum()); }
private LZ4Compressor(String type, Integer compressionLevel) { this.compressorType = type; this.compressionLevel = compressionLevel; final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); switch (type) { case LZ4_HIGH_COMPRESSOR: { compressor = lz4Factory.highCompressor(compressionLevel); break; } case LZ4_FAST_COMPRESSOR: default: { compressor = lz4Factory.fastCompressor(); } } decompressor = lz4Factory.fastDecompressor(); }
/** * Create a new {@link OutputStream} that will compress data using the LZ4 algorithm. * * @param out The output stream to compress * @param blockSize Default: 4. The block size used during compression. 4=64kb, 5=256kb, 6=1mb, 7=4mb. All other * values will generate an exception * @param blockChecksum Default: false. When true, a XXHash32 checksum is computed and appended to the stream for * every block of data * @param useBrokenFlagDescriptorChecksum Default: false. When true, writes an incorrect FrameDescriptor checksum * compatible with older kafka clients. * @throws IOException */ public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum, boolean useBrokenFlagDescriptorChecksum) throws IOException { this.out = out; compressor = LZ4Factory.fastestInstance().fastCompressor(); checksum = XXHashFactory.fastestInstance().hash32(); this.useBrokenFlagDescriptorChecksum = useBrokenFlagDescriptorChecksum; bd = new BD(blockSize); flg = new FLG(blockChecksum); bufferOffset = 0; maxBlockSize = bd.getBlockMaximumSize(); buffer = new byte[maxBlockSize]; compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)]; finished = false; writeHeader(); }
private LZ4Compressor() { final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); logger.info("Using {}", lz4Factory.toString()); compressor = lz4Factory.fastCompressor(); decompressor = lz4Factory.fastDecompressor(); }
public LZ4DataStore(IDataStore dataStore) { LZ4Factory factory = LZ4Factory.fastestInstance(); this.dataStore = dataStore; this.compressor = factory.fastCompressor(); this.decompressor = factory.fastDecompressor(); }
LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor(); Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum(); in = new DataInputStream(new LZ4BlockInputStream(socket.getInputStream(), decompressor,
LZ4Factory factory = LZ4Factory.fastestInstance(); byte[] data = "12345345234572".getBytes("UTF-8"); final int decompressedLength = data.length; // compress data LZ4Compressor compressor = factory.fastCompressor(); int maxCompressedLength = compressor.maxCompressedLength(decompressedLength); byte[] compressed = new byte[maxCompressedLength]; int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength); // decompress data // - method 1: when the decompressed length is known LZ4FastDecompressor decompressor = factory.fastDecompressor(); byte[] restored = new byte[decompressedLength]; int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength); // compressedLength == compressedLength2 // - method 2: when the compressed length is known (a little slower) // the destination buffer needs to be over-sized LZ4SafeDecompressor decompressor2 = factory.safeDecompressor(); int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0); // decompressedLength == decompressedLength2
public static byte[] compress(final byte[] src) { if (src == null) { throw new IllegalArgumentException("src must not be null."); } LZ4Compressor compressor = factory.fastCompressor(); return compressor.compress(src); }
public ClickHouseLZ4OutputStream(OutputStream stream, int maxCompressBlockSize) { dataWrapper = new LittleEndianDataOutputStream(stream); compressor = factory.fastCompressor(); currentBlock = new byte[maxCompressBlockSize]; compressedBlock = new byte[compressor.maxCompressedLength(maxCompressBlockSize)]; }
private LZ4Compressor() { final LZ4Factory lz4Factory = LZ4Factory.fastestInstance(); compressor = lz4Factory.fastCompressor(); decompressor = lz4Factory.decompressor(); }
/** * Creates a new customizable LZ4 decoder. * * @param factory user customizable {@link LZ4Factory} instance * which may be JNI bindings to the original C implementation, a pure Java implementation * or a Java implementation that uses the {@link sun.misc.Unsafe} * @param checksum the {@link Checksum} instance to use to check data for integrity. * You may set {@code null} if you do not want to validate checksum of each block */ public Lz4FrameDecoder(LZ4Factory factory, Checksum checksum) { if (factory == null) { throw new NullPointerException("factory"); } decompressor = factory.fastDecompressor(); this.checksum = checksum == null ? null : ByteBufChecksum.wrapChecksum(checksum); }
public static void encode(File file, File outputFile) { try { byte[] data= Files.readAllBytes(file.toPath()); LZ4Factory factory= LZ4Factory.safeInstance(); LZ4Compressor compressor= factory.highCompressor(9); FileOutputStream fos= new FileOutputStream(outputFile); byte[] result= compressor.compress(data); writeIntLE(data.length, fos); fos.write(result); fos.close(); } catch (Exception e) { throw new RuntimeException(e); } } }
@Override public byte[] compress(byte[] buf) { return LZ4Factory.fastestInstance().highCompressor().compress(buf); }
/** * Create a new {@link InputStream} that will decompress data using the LZ4 algorithm. * * @param in The stream to decompress * @throws IOException */ public LZ4FrameInputStream(InputStream in) throws IOException { this(in, LZ4Factory.fastestInstance().safeDecompressor(), XXHashFactory.fastestInstance().hash32()); }
/** * Create a new instance which compresses with the standard LZ4 compression * algorithm. * @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor) * @see LZ4Factory#fastCompressor() */ public LZ4BlockOutputStream(OutputStream out, int blockSize) { this(out, blockSize, LZ4Factory.fastestInstance().fastCompressor()); }
/** * Creates a LZ4 decoder with fastest decoder instance available on your machine. * * @param validateChecksums if {@code true}, the checksum field will be validated against the actual * uncompressed data, and if the checksums do not match, a suitable * {@link DecompressionException} will be thrown */ public Lz4FrameDecoder(boolean validateChecksums) { this(LZ4Factory.fastestInstance(), validateChecksums); }