/** * Create a {@link CompressionOutputStream} that will write to the given * {@link OutputStream} with the given {@link Compressor}. * * @param out the location for the final output stream * @param compressor compressor to use * @return a stream the user can write uncompressed data to have it compressed * @throws IOException */ @Override public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); int compressionOverhead = (bufferSize / 6) + 32; return new BlockCompressorStream(out, compressor, bufferSize, compressionOverhead); }
finish(); compressor.reset(); rawWriteInt(len); do { int bufLen = Math.min(len, MAX_INPUT_SIZE); compressor.finish(); while (!compressor.finished()) { compress(); rawWriteInt((int)compressor.getBytesRead()); do { compress(); } while (!compressor.needsInput());
@Override public void finish() throws IOException { if (!compressor.finished()) { rawWriteInt((int)compressor.getBytesRead()); compressor.finish(); while (!compressor.finished()) { compress(); } } }
@Test public void testRead() throws IOException { // compress empty stream bytesOut = new ByteArrayOutputStream(); BlockCompressorStream blockCompressorStream = new BlockCompressorStream(bytesOut, new FakeCompressor(), 1024, 0); // close without any write blockCompressorStream.close(); // check compressed output buf = bytesOut.toByteArray(); assertEquals("empty file compressed output size is not 4", 4, buf.length); // use compressed output as input for decompression bytesIn = new ByteArrayInputStream(buf); // get decompression stream BlockDecompressorStream blockDecompressorStream = new BlockDecompressorStream(bytesIn, new FakeDecompressor(), 1024); try { assertEquals("return value is not -1", -1 , blockDecompressorStream.read()); } catch (IOException e) { fail("unexpected IOException : " + e); } } }
@Override protected void compress() throws IOException { int len = compressor.compress(buffer, 0, buffer.length); if (len > 0) { // Write out the compressed chunk rawWriteInt(len); out.write(buffer, 0, len); } }
@Override public void finish() throws IOException { if (!compressor.finished()) { rawWriteInt((int)compressor.getBytesRead()); compressor.finish(); while (!compressor.finished()) { compress(); } } }
compressor.reset(); BlockCompressorStream blockCompressorStream = new BlockCompressorStream( bytesOut, compressor, 1024, 0); blockCompressorStream.close();
@Override protected void compress() throws IOException { int len = compressor.compress(buffer, 0, buffer.length); if (len > 0) { // Write out the compressed chunk rawWriteInt(len); out.write(buffer, 0, len); } }
finish(); compressor.reset(); rawWriteInt(len); do { int bufLen = Math.min(len, MAX_INPUT_SIZE); compressor.finish(); while (!compressor.finished()) { compress(); rawWriteInt((int)compressor.getBytesRead()); do { compress(); } while (!compressor.needsInput());
public void finish() throws IOException { if (!compressor.finished()) { rawWriteInt((int)compressor.getBytesRead()); compressor.finish(); while (!compressor.finished()) { compress(); } } }
compressor.reset(); BlockCompressorStream blockCompressorStream = new BlockCompressorStream( bytesOut, compressor, 1024, 0); blockCompressorStream.close();
/** * Create a {@link CompressionOutputStream} that will write to the given * {@link OutputStream} with the given {@link Compressor}. * * @param out the location for the final output stream * @param compressor compressor to use * @return a stream the user can write uncompressed data to have it compressed * @throws IOException */ @Override public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { if (!isNativeCodeLoaded()) { throw new RuntimeException("native lz4 library not available"); } int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT); int compressionOverhead = bufferSize/255 + 16; return new BlockCompressorStream(out, compressor, bufferSize, compressionOverhead); }
void compress() throws IOException { int len = compressor.compress(buffer, 0, buffer.length); if (len > 0) { // Write out the compressed chunk rawWriteInt(len); out.write(buffer, 0, len); } }
finish(); compressor.reset(); rawWriteInt(len); do { int bufLen = Math.min(len, MAX_INPUT_SIZE); compressor.finish(); while (!compressor.finished()) { compress(); rawWriteInt((int)compressor.getBytesRead()); do { compress(); } while (!compressor.needsInput());
@Override public void finish() throws IOException { if (!compressor.finished()) { rawWriteInt((int)compressor.getBytesRead()); compressor.finish(); while (!compressor.finished()) { compress(); } } }
new BlockCompressorStream(bytesOut, new FakeCompressor(), 1024, 0); blockCompressorStream.close();
/** * Create a {@link CompressionOutputStream} that will write to the given * {@link OutputStream} with the given {@link Compressor}. * * @param out the location for the final output stream * @param compressor compressor to use * @return a stream the user can write uncompressed data to have it compressed * @throws IOException */ @Override public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); int compressionOverhead = (bufferSize / 6) + 32; return new BlockCompressorStream(out, compressor, bufferSize, compressionOverhead); }
@Override protected void compress() throws IOException { int len = compressor.compress(buffer, 0, buffer.length); if (len > 0) { // Write out the compressed chunk rawWriteInt(len); out.write(buffer, 0, len); } }
finish(); compressor.reset(); rawWriteInt(len); do { int bufLen = Math.min(len, MAX_INPUT_SIZE); compressor.finish(); while (!compressor.finished()) { compress(); rawWriteInt((int)compressor.getBytesRead()); do { compress(); } while (!compressor.needsInput());
@Override public void finish() throws IOException { if (!compressor.finished()) { rawWriteInt((int)compressor.getBytesRead()); compressor.finish(); while (!compressor.finished()) { compress(); } } }