@Override public ByteBuffer compress(ByteBuffer in, ByteBuffer out) { out.clear(); int position = in.position(); lz4High.compress(in, out); in.position(position); out.flip(); return out; } }
private byte[] serialize(byte[] value) { final int len = LZ4_COMPRESSOR.maxCompressedLength(value.length); final byte[] out = new byte[len]; final int compressedSize = LZ4_COMPRESSOR.compress(value, 0, value.length, out, 0); return ByteBuffer.allocate(compressedSize + Integer.BYTES) .putInt(value.length) .put(out, 0, compressedSize) .array(); } }
@Override public byte[] compress(byte[] data, int offset, int length, OpStatsLogger compressionStat) { Preconditions.checkNotNull(data); Preconditions.checkArgument(offset >= 0 && offset < data.length); Preconditions.checkArgument(length >= 0); Preconditions.checkNotNull(compressionStat); Stopwatch watch = Stopwatch.createStarted(); byte[] compressed = compressor.compress(data, offset, length); compressionStat.registerSuccessfulEvent(watch.elapsed(TimeUnit.MICROSECONDS)); return compressed; }
@Override public ByteBuf encode(ByteBuf source) { int uncompressedLength = source.readableBytes(); int maxLength = compressor.maxCompressedLength(uncompressedLength); ByteBuffer sourceNio = source.nioBuffer(source.readerIndex(), source.readableBytes()); ByteBuf target = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength); ByteBuffer targetNio = target.nioBuffer(0, maxLength); int compressedLength = compressor.compress(sourceNio, 0, uncompressedLength, targetNio, 0, maxLength); target.writerIndex(compressedLength); return target; }
@Override protected byte[] compress(byte[] in) { if (in == null) { throw new NullPointerException("Can't compress null"); } LZ4Compressor compressor = lz4Factory.fastCompressor(); byte[] out = new byte[compressor.maxCompressedLength(in.length)]; int compressedLength = compressor.compress(in, 0, in.length, out, 0); getLogger().debug("Compressed %d bytes to %d", in.length, compressedLength); return ByteBuffer.allocate(Integer.BYTES + compressedLength) .putInt(in.length) .put(out, 0, compressedLength) .array(); }
@Override public ByteBuf encode(Object in) throws IOException { ByteBuf bytes = null; try { LZ4Compressor compressor = factory.fastCompressor(); bytes = innerCodec.getValueEncoder().encode(in); ByteBuffer srcBuf = bytes.internalNioBuffer(bytes.readerIndex(), bytes.readableBytes()); int outMaxLength = compressor.maxCompressedLength(bytes.readableBytes()); ByteBuf out = ByteBufAllocator.DEFAULT.buffer(outMaxLength + DECOMPRESSION_HEADER_SIZE); out.writeInt(bytes.readableBytes()); ByteBuffer outBuf = out.internalNioBuffer(out.writerIndex(), out.writableBytes()); int pos = outBuf.position(); compressor.compress(srcBuf, outBuf); int compressedLength = outBuf.position() - pos; out.writerIndex(out.writerIndex() + compressedLength); return out; } finally { if (bytes != null) { bytes.release(); } } } };
@Override public ByteBuf encode(Object in) throws IOException { ByteBuf bytes = null; try { LZ4Compressor compressor = factory.fastCompressor(); bytes = innerCodec.getValueEncoder().encode(in); ByteBuffer srcBuf = bytes.internalNioBuffer(bytes.readerIndex(), bytes.readableBytes()); int outMaxLength = compressor.maxCompressedLength(bytes.readableBytes()); ByteBuf out = ByteBufAllocator.DEFAULT.buffer(outMaxLength + DECOMPRESSION_HEADER_SIZE); out.writeInt(bytes.readableBytes()); ByteBuffer outBuf = out.internalNioBuffer(out.writerIndex(), out.writableBytes()); int pos = outBuf.position(); compressor.compress(srcBuf, outBuf); int compressedLength = outBuf.position() - pos; out.writerIndex(out.writerIndex() + compressedLength); return out; } finally { if (bytes != null) { bytes.release(); } } } };
/** * @param compactPage Compacted page. * @param compactSize Compacted page size. * @param compressLevel Compression level. * @return Compressed page. */ private ByteBuffer compressPageLz4(ByteBuffer compactPage, int compactSize, int compressLevel) { LZ4Compressor compressor = Lz4.getCompressor(compressLevel); ByteBuffer compressedPage = compressBuf.get(); copyPageHeader(compactPage, compressedPage, compactSize); compressor.compress(compactPage, compressedPage); compactPage.flip(); compressedPage.flip(); return compressedPage; }
/** * Compresses buffered data, optionally computes an XXHash32 checksum, and writes the result to the underlying * {@link OutputStream}. * * @throws IOException */ private void writeBlock() throws IOException { if (bufferOffset == 0) { return; } int compressedLength = compressor.compress(buffer, 0, bufferOffset, compressedBuffer, 0); byte[] bufferToWrite = compressedBuffer; int compressMethod = 0; // Store block uncompressed if compressed length is greater (incompressible) if (compressedLength >= bufferOffset) { bufferToWrite = buffer; compressedLength = bufferOffset; compressMethod = LZ4_FRAME_INCOMPRESSIBLE_MASK; } // Write content ByteUtils.writeUnsignedIntLE(out, compressedLength | compressMethod); out.write(bufferToWrite, 0, compressedLength); // Calculate and write block checksum if (flg.isBlockChecksumSet()) { int hash = checksum.hash(bufferToWrite, 0, compressedLength, 0); ByteUtils.writeUnsignedIntLE(out, hash); } bufferOffset = 0; }
int pos = outNioBuffer.position(); compressor.compress(buffer.internalNioBuffer(buffer.readerIndex(), flushableBytes), outNioBuffer); compressedLength = outNioBuffer.position() - pos; } catch (LZ4Exception e) {
int pos = outNioBuffer.position(); compressor.compress(buffer.internalNioBuffer(buffer.readerIndex(), flushableBytes), outNioBuffer); compressedLength = outNioBuffer.position() - pos; } catch (LZ4Exception e) {
int pos = outNioBuffer.position(); compressor.compress(buffer.internalNioBuffer(buffer.readerIndex(), flushableBytes), outNioBuffer); compressedLength = outNioBuffer.position() - pos; } catch (LZ4Exception e) {
private ByteBuf compressDirect(ByteBuf input) throws IOException { int maxCompressedLength = compressor.maxCompressedLength(input.readableBytes()); // If the input is direct we will allocate a direct output buffer as well as this will allow us // to use // LZ4Compressor.compress and so eliminate memory copies. ByteBuf output = input.alloc().directBuffer(INTEGER_BYTES + maxCompressedLength); try { ByteBuffer in = inputNioBuffer(input); // Increase reader index. input.readerIndex(input.writerIndex()); output.writeInt(in.remaining()); ByteBuffer out = outputNioBuffer(output); int written = compressor.compress( in, in.position(), in.remaining(), out, out.position(), out.remaining()); // Set the writer index so the amount of written bytes is reflected output.writerIndex(output.writerIndex() + written); } catch (Exception e) { // release output buffer so we not leak and rethrow exception. output.release(); throw new IOException(e); } return output; }
private ByteBuf compressHeap(ByteBuf input) throws IOException { int maxCompressedLength = compressor.maxCompressedLength(input.readableBytes()); // Not a direct buffer so use byte arrays... int inOffset = input.arrayOffset() + input.readerIndex(); byte[] in = input.array(); int len = input.readableBytes(); // Increase reader index. input.readerIndex(input.writerIndex()); // Allocate a heap buffer from the ByteBufAllocator as we may use a PooledByteBufAllocator and // so // can eliminate the overhead of allocate a new byte[]. ByteBuf output = input.alloc().heapBuffer(INTEGER_BYTES + maxCompressedLength); try { output.writeInt(len); // calculate the correct offset. int offset = output.arrayOffset() + output.writerIndex(); byte[] out = output.array(); int written = compressor.compress(in, inOffset, len, out, offset); // Set the writer index so the amount of written bytes is reflected output.writerIndex(output.writerIndex() + written); } catch (Exception e) { // release output buffer so we not leak and rethrow exception. output.release(); throw new IOException(e); } return output; }
@Override public int compress(ByteBuffer src, ByteBuffer target) { int targetPosition = target.position(); compressor.compress(src, target); return target.position() - targetPosition; }
@Override public ByteBuffer compress(ByteBuffer in, ByteBuffer out) { out.clear(); int position = in.position(); lz4High.compress(in, out); in.position(position); out.flip(); return out; } }
/** * Compress <code>src</code> into <code>dest</code>. Calling this method * will update the positions of both {@link ByteBuffer}s. */ public final void compress(ByteBuffer src, ByteBuffer dest) { final int cpLen = compress(src, src.position(), src.remaining(), dest, dest.position(), dest.remaining()); src.position(src.limit()); dest.position(dest.position() + cpLen); }
@Override public int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength) { return compressor.compress(input, inputOffset, inputLength, output, outputOffset, maxOutputLength); }
private byte[] serialize(byte[] value) { final int len = LZ4_COMPRESSOR.maxCompressedLength(value.length); final byte[] out = new byte[len]; final int compressedSize = LZ4_COMPRESSOR.compress(value, 0, value.length, out, 0); return ByteBuffer.allocate(compressedSize + Integer.BYTES) .putInt(value.length) .put(out, 0, compressedSize) .array(); } }
@Override public byte[] compress(byte[] data, int offset, int length, OpStatsLogger compressionStat) { Preconditions.checkNotNull(data); Preconditions.checkArgument(offset >= 0 && offset < data.length); Preconditions.checkArgument(length >= 0); Preconditions.checkNotNull(compressionStat); Stopwatch watch = Stopwatch.createStarted(); byte[] compressed = compressor.compress(data, offset, length); compressionStat.registerSuccessfulEvent(watch.elapsed(TimeUnit.MICROSECONDS)); return compressed; }