@Override public int decompress(byte[] input, int offset, int length, OutputBuffer output) throws OrcCorruptionException { try { int uncompressedLength = SnappyDecompressor.getUncompressedLength(input, offset); if (uncompressedLength > maxBufferSize) { throw new OrcCorruptionException(orcDataSourceId, "Snappy requires buffer (%s) larger than max size (%s)", uncompressedLength, maxBufferSize); } // Snappy decompressor is more efficient if there's at least a long's worth of extra space // in the output buffer byte[] buffer = output.initialize(uncompressedLength + SIZE_OF_LONG); return decompressor.decompress(input, offset, length, buffer, 0, buffer.length); } catch (MalformedInputException e) { throw new OrcCorruptionException(e, orcDataSourceId, "Invalid compressed stream"); } }
private static Slice decompressSnappy(Slice input, int uncompressedSize) { byte[] buffer = new byte[uncompressedSize]; decompress(new SnappyDecompressor(), input, 0, input.length(), buffer, 0); return wrappedBuffer(buffer); }
@Override public RcFileCompressor createCompressor(String codecName) { if (SNAPPY_CODEC_NAME.equals(codecName)) { return new AircompressorCompressor(new SnappyCodec()); } if (LZO_CODEC_NAME.equals(codecName) || LZO_CODEC_NAME_DEPRECATED.equals(codecName)) { return new AircompressorCompressor(new LzoCodec()); } if (LZ4_CODEC_NAME.equals(codecName)) { return new AircompressorCompressor(new Lz4Codec()); } if (GZIP_CODEC_NAME.equals(codecName)) { return new AircompressorCompressor(new JdkGzipCodec()); } return delegate.createCompressor(codecName); }
SnappyCodec() { super(new SnappyCompressor(), new SnappyDecompressor()); }
public OrcOutputBuffer(CompressionKind compression, int maxBufferSize) { requireNonNull(compression, "compression is null"); checkArgument(maxBufferSize > 0, "maximum buffer size should be greater than 0"); this.maxBufferSize = maxBufferSize; this.buffer = new byte[INITIAL_BUFFER_SIZE]; this.slice = wrappedBuffer(buffer); compressedOutputStream = new ChunkedSliceOutput(MINIMUM_OUTPUT_BUFFER_CHUNK_SIZE, MAXIMUM_OUTPUT_BUFFER_CHUNK_SIZE); if (compression == CompressionKind.NONE) { this.compressor = null; } else if (compression == CompressionKind.SNAPPY) { this.compressor = new SnappyCompressor(); } else if (compression == CompressionKind.ZLIB) { this.compressor = new DeflateCompressor(); } else if (compression == CompressionKind.LZ4) { this.compressor = new Lz4Compressor(); } else if (compression == CompressionKind.ZSTD) { this.compressor = new ZstdJniCompressor(); } else { throw new IllegalArgumentException("Unsupported compression " + compression); } }
public HadoopSnappyOutputStream(OutputStream out, int bufferSize) { super(out); inputBuffer = new byte[bufferSize]; // leave extra space free at end of buffers to make compression (slightly) faster inputMaxSize = inputBuffer.length - compressionOverhead(bufferSize); outputBuffer = new byte[compressor.maxCompressedLength(inputMaxSize) + SIZE_OF_LONG]; }
public static int maskedCrc32c(byte[] data, int offset, int length) { Crc32C crc32c = new Crc32C(); crc32c.update(data, offset, length); return crc32c.getMaskedValue(); }
@Override public CompressionOutputStream createOutputStream(OutputStream outputStream) throws IOException { return new HadoopSnappyOutputStream(outputStream, getBufferSize()); }
/** * Creates a Snappy output stream with block checksums disabled. This is only useful for * apples-to-apples benchmarks with other compressors that do not perform block checksums. * * @param out the underlying output stream */ public static SnappyFramedOutputStream newChecksumFreeBenchmarkOutputStream(OutputStream out) throws IOException { return new SnappyFramedOutputStream(out, false); }
public static int getUncompressedLength(byte[] compressed, int compressedOffset) { long compressedAddress = ARRAY_BYTE_BASE_OFFSET + compressedOffset; long compressedLimit = ARRAY_BYTE_BASE_OFFSET + compressed.length; return SnappyRawDecompressor.getUncompressedLength(compressed, compressedAddress, compressedLimit); }
@Override public int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength) { long inputAddress = ARRAY_BYTE_BASE_OFFSET + inputOffset; long inputLimit = inputAddress + inputLength; long outputAddress = ARRAY_BYTE_BASE_OFFSET + outputOffset; long outputLimit = outputAddress + maxOutputLength; return SnappyRawCompressor.compress(input, inputAddress, inputLimit, output, outputAddress, outputLimit, table); }
@Override public Compressor createCompressor() { return new HadoopSnappyCompressor(); }
@Override public Decompressor createDecompressor() { return new HadoopSnappyDecompressor(); }
@Override public int read() throws IOException { if (uncompressedChunkOffset >= uncompressedChunkLength) { readNextChunk(uncompressedChunk, 0, uncompressedChunk.length); if (uncompressedChunkLength == 0) { return -1; } } return uncompressedChunk[uncompressedChunkOffset++] & 0xFF; }
@Override public int read() throws IOException { if (closed) { return -1; } if (!ensureBuffer()) { return -1; } return buffer[position++] & 0xFF; }
@Override public void write(int b) throws IOException { inputBuffer[inputOffset++] = (byte) b; if (inputOffset >= inputMaxSize) { writeNextChunk(inputBuffer, 0, this.inputOffset); } }
@Override protected Decompressor getDecompressor() { return new SnappyDecompressor(); }
@Override public RcFileDecompressor createDecompressor(String codecName) { if (SNAPPY_CODEC_NAME.equals(codecName)) { return new AircompressorDecompressor(new SnappyCodec()); } if (LZO_CODEC_NAME.equals(codecName) || LZO_CODEC_NAME_DEPRECATED.equals(codecName)) { return new AircompressorDecompressor(new LzoCodec()); } if (LZ4_CODEC_NAME.equals(codecName) || LZ4_HC_CODEC_NAME.equals(codecName)) { return new AircompressorDecompressor(new Lz4Codec()); } if (GZIP_CODEC_NAME.equals(codecName)) { return new AircompressorDecompressor(new JdkGzipCodec()); } return delegate.createDecompressor(codecName); } }
public HadoopSnappyOutputStream(OutputStream out, int bufferSize) { super(out); inputBuffer = new byte[bufferSize]; // leave extra space free at end of buffers to make compression (slightly) faster inputMaxSize = inputBuffer.length - compressionOverhead(bufferSize); outputBuffer = new byte[compressor.maxCompressedLength(inputMaxSize) + SIZE_OF_LONG]; }
@Override public CompressionOutputStream createOutputStream(OutputStream outputStream) throws IOException { return new HadoopSnappyOutputStream(outputStream, getBufferSize()); }