public SerializedPage serialize(Page page) { SliceOutput serializationBuffer = new DynamicSliceOutput(toIntExact((page.getSizeInBytes() + Integer.BYTES))); // block length is an int writeRawPage(page, serializationBuffer, blockEncodingSerde); if (!compressor.isPresent()) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } int maxCompressedLength = maxCompressedLength(serializationBuffer.size()); byte[] compressionBuffer = new byte[maxCompressedLength]; int actualCompressedLength = compressor.get().compress(serializationBuffer.slice().getBytes(), 0, serializationBuffer.size(), compressionBuffer, 0, maxCompressedLength); if (((1.0 * actualCompressedLength) / serializationBuffer.size()) > MINIMUM_COMPRESSION_RATIO) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } return new SerializedPage( Slices.copyOf(Slices.wrappedBuffer(compressionBuffer, 0, actualCompressedLength)), COMPRESSED, page.getPositionCount(), serializationBuffer.size()); }
@Override public boolean compress(ByteBuffer in, ByteBuffer out, ByteBuffer overflow) throws IOException { int inBytes = in.remaining(); // I should work on a patch for Snappy to support an overflow buffer // to prevent the extra buffer copy. byte[] compressed = getBuffer(compressor.maxCompressedLength(inBytes)); int outBytes = compressor.compress(in.array(), in.arrayOffset() + in.position(), inBytes, compressed, 0, compressed.length); if (outBytes < inBytes) { int remaining = out.remaining(); if (remaining >= outBytes) { System.arraycopy(compressed, 0, out.array(), out.arrayOffset() + out.position(), outBytes); out.position(out.position() + outBytes); } else { System.arraycopy(compressed, 0, out.array(), out.arrayOffset() + out.position(), remaining); out.position(out.limit()); System.arraycopy(compressed, remaining, overflow.array(), overflow.arrayOffset(), outBytes - remaining); overflow.position(outBytes - remaining); } return true; } else { return false; } }
@Override public int maxCompressedLength(int uncompressedSize) { // assume hadoop stream encoder won't increase size by more than 10% over the block encoder return (int) ((blockCompressorForSizeCalculation.maxCompressedLength(uncompressedSize) * 1.1) + 8); }
private byte[] prepareCompressedData(byte[] uncompressed) { Compressor compressor = getVerifyCompressor(); byte[] compressed = new byte[compressor.maxCompressedLength(uncompressed.length)]; int compressedLength = compressor.compress( uncompressed, 0, uncompressed.length, compressed, 0, compressed.length); return Arrays.copyOf(compressed, compressedLength); } }
@Test(dataProvider = "data") public void testCompressByteBufferHeapToDirect(DataSet dataSet) throws Exception { if (!isByteBufferSupported()) { return; } byte[] uncompressedOriginal = dataSet.getUncompressed(); Compressor compressor = getCompressor(); verifyCompressByteBuffer( compressor, ByteBuffer.wrap(uncompressedOriginal), ByteBuffer.allocateDirect(compressor.maxCompressedLength(uncompressedOriginal.length))); }
@Test(dataProvider = "data") public void testCompress(DataSet testCase) throws Exception { Compressor compressor = getCompressor(); byte[] originalUncompressed = testCase.getUncompressed(); byte[] compressed = new byte[compressor.maxCompressedLength(originalUncompressed.length)]; // attempt to compress slightly different data to ensure the compressor doesn't keep state // between calls that may affect results if (originalUncompressed.length > 1) { byte[] output = new byte[compressor.maxCompressedLength(originalUncompressed.length - 1)]; compressor.compress(originalUncompressed, 1, originalUncompressed.length - 1, output, 0, output.length); } int compressedLength = compressor.compress( originalUncompressed, 0, originalUncompressed.length, compressed, 0, compressed.length); verifyCompressedData(originalUncompressed, compressed, compressedLength); }
@Benchmark public int compress(BytesCounter counter) { int written = compressor.compress(uncompressed, 0, uncompressed.length, compressTarget, 0, compressTarget.length); counter.bytes += uncompressed.length; return written; }
@Test(dataProvider = "data") public void testCompressByteBufferHeapToHeap(DataSet dataSet) throws Exception { if (!isByteBufferSupported()) { return; } byte[] uncompressedOriginal = dataSet.getUncompressed(); Compressor compressor = getCompressor(); verifyCompressByteBuffer( compressor, ByteBuffer.wrap(uncompressedOriginal), ByteBuffer.allocate(compressor.maxCompressedLength(uncompressedOriginal.length))); }
@Setup public void setup(DataSet data) throws IOException { uncompressed = data.getUncompressed(); compressor = algorithm.getCompressor(); compressTarget = new byte[compressor.maxCompressedLength(uncompressed.length)]; decompressor = algorithm.getDecompressor(); Compressor compressor = algorithm.getCompressor(); compressed = new byte[compressor.maxCompressedLength(uncompressed.length)]; int compressedLength = compressor.compress(uncompressed, 0, uncompressed.length, compressed, 0, compressed.length); compressed = Arrays.copyOf(compressed, compressedLength); uncompressTarget = new byte[uncompressed.length]; }
private void verifyCompressByteBuffer(Compressor compressor, ByteBuffer expected, ByteBuffer compressed) { // attempt to compress slightly different data to ensure the compressor doesn't keep state // between calls that may affect results if (expected.remaining() > 1) { ByteBuffer duplicate = expected.duplicate(); duplicate.get(); // skip one byte compressor.compress(duplicate, ByteBuffer.allocate(compressed.remaining())); } compressor.compress(expected.duplicate(), compressed); compressed.flip(); ByteBuffer uncompressed = ByteBuffer.allocate(expected.remaining()); // TODO: validate with "control" decompressor getDecompressor().decompress(compressed, uncompressed); uncompressed.flip(); assertByteBufferEqual(expected.duplicate(), uncompressed); }
@Test(dataProvider = "data") public void testCompressByteBufferDirectToHeap(DataSet dataSet) throws Exception { if (!isByteBufferSupported()) { return; } byte[] uncompressedOriginal = dataSet.getUncompressed(); Compressor compressor = getCompressor(); verifyCompressByteBuffer( compressor, toDirectBuffer(uncompressedOriginal), ByteBuffer.allocate(compressor.maxCompressedLength(uncompressedOriginal.length))); }
@Test public void testRoundTripSmallLiteral() throws Exception { byte[] data = new byte[256]; for (int i = 0; i < data.length; i++) { data[i] = (byte) i; } Compressor compressor = getCompressor(); byte[] compressed = new byte[compressor.maxCompressedLength(data.length)]; byte[] uncompressed = new byte[data.length]; for (int i = 1; i < data.length; i++) { try { int written = compressor.compress( data, 0, i, compressed, 0, compressed.length); int decompressedSize = getDecompressor().decompress(compressed, 0, written, uncompressed, 0, uncompressed.length); assertByteArraysEqual(data, 0, i, uncompressed, 0, decompressedSize); assertEquals(decompressedSize, i); } catch (MalformedInputException e) { throw new RuntimeException("Failed with " + i + " bytes of input", e); } } }
public SerializedPage serialize(Page page) { SliceOutput serializationBuffer = new DynamicSliceOutput(toIntExact((page.getSizeInBytes() + Integer.BYTES))); // block length is an int writeRawPage(page, serializationBuffer, blockEncodingSerde); if (!compressor.isPresent()) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } int maxCompressedLength = maxCompressedLength(serializationBuffer.size()); byte[] compressionBuffer = new byte[maxCompressedLength]; int actualCompressedLength = compressor.get().compress(serializationBuffer.slice().getBytes(), 0, serializationBuffer.size(), compressionBuffer, 0, maxCompressedLength); if (((1.0 * actualCompressedLength) / serializationBuffer.size()) > MINIMUM_COMPRESSION_RATIO) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } return new SerializedPage( Slices.copyOf(Slices.wrappedBuffer(compressionBuffer, 0, actualCompressedLength)), COMPRESSED, page.getPositionCount(), serializationBuffer.size()); }
@Test(dataProvider = "data") public void testCompressByteBufferDirectToDirect(DataSet dataSet) throws Exception { if (!isByteBufferSupported()) { return; } byte[] uncompressedOriginal = dataSet.getUncompressed(); Compressor compressor = getCompressor(); verifyCompressByteBuffer( compressor, toDirectBuffer(uncompressedOriginal), ByteBuffer.allocateDirect(compressor.maxCompressedLength(uncompressedOriginal.length))); }
private static int compressSize(String algorithmName, String name) { try { Compressor compressor = Algorithm.valueOf(algorithmName).getCompressor(); DataSet dataSet = new DataSet(name); dataSet.loadFile(); byte[] uncompressed = dataSet.getUncompressed(); byte[] compressed = new byte[compressor.maxCompressedLength(uncompressed.length)]; return compressor.compress(uncompressed, 0, uncompressed.length, compressed, 0, compressed.length); } catch (Exception e) { return -1; } } }
public SerializedPage serialize(Page page) { SliceOutput serializationBuffer = new DynamicSliceOutput(toIntExact((page.getSizeInBytes() + Integer.BYTES))); // block length is an int writeRawPage(page, serializationBuffer, blockEncodingSerde); if (!compressor.isPresent()) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } int maxCompressedLength = maxCompressedLength(serializationBuffer.size()); byte[] compressionBuffer = new byte[maxCompressedLength]; int actualCompressedLength = compressor.get().compress(serializationBuffer.slice().getBytes(), 0, serializationBuffer.size(), compressionBuffer, 0, maxCompressedLength); if (((1.0 * actualCompressedLength) / serializationBuffer.size()) > MINIMUM_COMPRESSION_RATIO) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } return new SerializedPage( Slices.copyOf(Slices.wrappedBuffer(compressionBuffer, 0, actualCompressedLength)), COMPRESSED, page.getPositionCount(), serializationBuffer.size()); }