static void doCodec(final Codec codec, final Cell [] cells, final int cycles, final int count, final int initialBufferSize) throws IOException { byte [] bytes = null; Cell [] cellsDecoded = null; for (int i = 0; i < cycles; i++) { ByteArrayOutputStream baos = new ByteArrayOutputStream(initialBufferSize); Codec.Encoder encoder = codec.getEncoder(baos); bytes = runEncoderTest(i, initialBufferSize, baos, encoder, cells); } for (int i = 0; i < cycles; i++) { ByteArrayInputStream bais = new ByteArrayInputStream(bytes); Codec.Decoder decoder = codec.getDecoder(bais); cellsDecoded = CodecPerformance.runDecoderTest(i, count, decoder); } verifyCells(cells, cellsDecoded); }
/** * @param codec to use for cellblock * @param cellBlock ByteBuffer containing the cells written by the Codec. The buffer should be * position()'ed at the start of the cell block and limit()'ed at the end. * @return CellScanner to work against the content of <code>cellBlock</code>. All cells created * out of the CellScanner will share the same ByteBuffer being passed. * @throws IOException if cell encoding fails */ public CellScanner createCellScannerReusingBuffers(final Codec codec, final CompressionCodec compressor, ByteBuff cellBlock) throws IOException { // Use this method from HRS to create the CellScanner // If compressed, decompress it first before passing it on else we will leak compression // resources if the stream is not closed properly after we let it out. if (compressor != null) { cellBlock = decompress(compressor, cellBlock); } return codec.getDecoder(cellBlock); }
private void encodeCellsTo(OutputStream os, CellScanner cellScanner, Codec codec, CompressionCodec compressor) throws IOException { Compressor poolCompressor = null; try { if (compressor != null) { if (compressor instanceof Configurable) { ((Configurable) compressor).setConf(this.conf); } poolCompressor = CodecPool.getCompressor(compressor); os = compressor.createOutputStream(os, poolCompressor); } Codec.Encoder encoder = codec.getEncoder(os); while (cellScanner.advance()) { encoder.write(cellScanner.current()); } encoder.flush(); } catch (BufferOverflowException | IndexOutOfBoundsException e) { throw new DoNotRetryIOException(e); } finally { os.close(); if (poolCompressor != null) { CodecPool.returnCompressor(poolCompressor); } } }
/** * @param codec to use for cellblock * @param cellBlock to encode * @return CellScanner to work against the content of <code>cellBlock</code> * @throws IOException if encoding fails */ public CellScanner createCellScanner(final Codec codec, final CompressionCodec compressor, final byte[] cellBlock) throws IOException { // Use this method from Client side to create the CellScanner if (compressor != null) { ByteBuffer cellBlockBuf = decompress(compressor, cellBlock); return codec.getDecoder(new ByteBufferInputStream(cellBlockBuf)); } // Not making the Decoder over the ByteBuffer purposefully. The Decoder over the BB will // make Cells directly over the passed BB. This method is called at client side and we don't // want the Cells to share the same byte[] where the RPC response is being read. Caching of any // of the Cells at user's app level will make it not possible to GC the response byte[] return codec.getDecoder(new ByteArrayInputStream(cellBlock)); }
private void encodeCellsTo(OutputStream os, CellScanner cellScanner, Codec codec, CompressionCodec compressor) throws IOException { Compressor poolCompressor = null; try { if (compressor != null) { if (compressor instanceof Configurable) { ((Configurable) compressor).setConf(this.conf); } poolCompressor = CodecPool.getCompressor(compressor); os = compressor.createOutputStream(os, poolCompressor); } Codec.Encoder encoder = codec.getEncoder(os); while (cellScanner.advance()) { encoder.write(cellScanner.current()); } encoder.flush(); } catch (BufferOverflowException | IndexOutOfBoundsException e) { throw new DoNotRetryIOException(e); } finally { os.close(); if (poolCompressor != null) { CodecPool.returnCompressor(poolCompressor); } } }
@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }
/** * @param codec to use for cellblock * @param cellBlock ByteBuffer containing the cells written by the Codec. The buffer should be * position()'ed at the start of the cell block and limit()'ed at the end. * @return CellScanner to work against the content of <code>cellBlock</code>. All cells created * out of the CellScanner will share the same ByteBuffer being passed. * @throws IOException if cell encoding fails */ public CellScanner createCellScannerReusingBuffers(final Codec codec, final CompressionCodec compressor, ByteBuff cellBlock) throws IOException { // Use this method from HRS to create the CellScanner // If compressed, decompress it first before passing it on else we will leak compression // resources if the stream is not closed properly after we let it out. if (compressor != null) { cellBlock = decompress(compressor, cellBlock); } return codec.getDecoder(cellBlock); }
private void encodeCellsTo(OutputStream os, CellScanner cellScanner, Codec codec, CompressionCodec compressor) throws IOException { Compressor poolCompressor = null; try { if (compressor != null) { if (compressor instanceof Configurable) { ((Configurable) compressor).setConf(this.conf); } poolCompressor = CodecPool.getCompressor(compressor); os = compressor.createOutputStream(os, poolCompressor); } Codec.Encoder encoder = codec.getEncoder(os); while (cellScanner.advance()) { encoder.write(cellScanner.current()); } encoder.flush(); } catch (BufferOverflowException | IndexOutOfBoundsException e) { throw new DoNotRetryIOException(e); } finally { os.close(); if (poolCompressor != null) { CodecPool.returnCompressor(poolCompressor); } } }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); kv.setSequenceId(Long.MAX_VALUE); encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV // Second read should trip over the end-of-stream marker and return false assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount()); }
/** * @param codec to use for cellblock * @param cellBlock to encode * @return CellScanner to work against the content of <code>cellBlock</code> * @throws IOException if encoding fails */ public CellScanner createCellScanner(final Codec codec, final CompressionCodec compressor, final byte[] cellBlock) throws IOException { // Use this method from Client side to create the CellScanner if (compressor != null) { ByteBuffer cellBlockBuf = decompress(compressor, cellBlock); return codec.getDecoder(new ByteBufferInputStream(cellBlockBuf)); } // Not making the Decoder over the ByteBuffer purposefully. The Decoder over the BB will // make Cells directly over the passed BB. This method is called at client side and we don't // want the Cells to share the same byte[] where the RPC response is being read. Caching of any // of the Cells at user's app level will make it not possible to GC the response byte[] return codec.getDecoder(new ByteArrayInputStream(cellBlock)); }
os = compressor.createOutputStream(os, poolCompressor); Codec.Encoder encoder = codec.getEncoder(os); int count = 0; while (cellScanner.advance()) {
DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); final KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), Bytes.toBytes("1")); new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); Cell c = decoder.current();
/** * @param codec to use for cellblock * @param cellBlock ByteBuffer containing the cells written by the Codec. The buffer should be * position()'ed at the start of the cell block and limit()'ed at the end. * @return CellScanner to work against the content of <code>cellBlock</code>. All cells created * out of the CellScanner will share the same ByteBuffer being passed. * @throws IOException if cell encoding fails */ public CellScanner createCellScannerReusingBuffers(final Codec codec, final CompressionCodec compressor, ByteBuff cellBlock) throws IOException { // Use this method from HRS to create the CellScanner // If compressed, decompress it first before passing it on else we will leak compression // resources if the stream is not closed properly after we let it out. if (compressor != null) { cellBlock = decompress(compressor, cellBlock); } return codec.getDecoder(cellBlock); }
DataOutputStream dos = new DataOutputStream(cos); Codec codec = new KeyValueCodecWithTags(); Codec.Encoder encoder = codec.getEncoder(dos); final KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("1"), new Tag[] { CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); Cell c = decoder.current();
/** * @param codec to use for cellblock * @param cellBlock to encode * @return CellScanner to work against the content of <code>cellBlock</code> * @throws IOException if encoding fails */ public CellScanner createCellScanner(final Codec codec, final CompressionCodec compressor, final byte[] cellBlock) throws IOException { // Use this method from Client side to create the CellScanner if (compressor != null) { ByteBuffer cellBlockBuf = decompress(compressor, cellBlock); return codec.getDecoder(new ByteBufferInputStream(cellBlockBuf)); } // Not making the Decoder over the ByteBuffer purposefully. The Decoder over the BB will // make Cells directly over the passed BB. This method is called at client side and we don't // want the Cells to share the same byte[] where the RPC response is being read. Caching of any // of the Cells at user's app level will make it not possible to GC the response byte[] return codec.getDecoder(new ByteArrayInputStream(cellBlock)); }
DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodecWithTags(); Codec.Encoder encoder = codec.getEncoder(dos); final Cell cell1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("1"), new Tag[] { CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); Cell c = decoder.current();
is = new ByteArrayInputStream(cellBlock, offset, length); return codec.getDecoder(is);
static void doCodec(final Codec codec, final Cell [] cells, final int cycles, final int count, final int initialBufferSize) throws IOException { byte [] bytes = null; Cell [] cellsDecoded = null; for (int i = 0; i < cycles; i++) { ByteArrayOutputStream baos = new ByteArrayOutputStream(initialBufferSize); Codec.Encoder encoder = codec.getEncoder(baos); bytes = runEncoderTest(i, initialBufferSize, baos, encoder, cells); } for (int i = 0; i < cycles; i++) { ByteArrayInputStream bais = new ByteArrayInputStream(bytes); Codec.Decoder decoder = codec.getDecoder(bais); cellsDecoded = CodecPerformance.runDecoderTest(i, count, decoder); } verifyCells(cells, cellsDecoded); }
@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }
@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }