private void testBBPool(int maxPoolSize, int bufferSize, boolean directByteBuffer) { ByteBufferPool pool = new ByteBufferPool(bufferSize, maxPoolSize, directByteBuffer); for (int i = 0; i < maxPoolSize; i++) { ByteBuffer buffer = pool.getBuffer(); assertEquals(0, buffer.position()); assertEquals(bufferSize, buffer.limit()); assertEquals(directByteBuffer, buffer.isDirect()); } assertEquals(0, pool.getQueueSize()); ByteBuffer bb = directByteBuffer ? ByteBuffer.allocate(bufferSize) : ByteBuffer.allocateDirect(bufferSize); pool.putbackBuffer(bb); assertEquals(0, pool.getQueueSize()); bb = directByteBuffer ? ByteBuffer.allocateDirect(bufferSize + 1) : ByteBuffer.allocate(bufferSize + 1); pool.putbackBuffer(bb); assertEquals(0, pool.getQueueSize()); } }
int minSizeForPoolUse, int reqLen) { ByteBuff resultBuf; List<ByteBuffer> bbs = new ArrayList<>((reqLen / pool.getBufferSize()) + 1); int remain = reqLen; ByteBuffer buf = null; while (remain >= minSizeForPoolUse && (buf = pool.getBuffer()) != null) { bbs.add(buf); remain -= pool.getBufferSize(); pool.putbackBuffer(bufsFromPoolFinal[i]);
private void allocateNewBuffer() { if (this.curBuf != null) { this.curBuf.flip();// On the current buf set limit = pos and pos = 0. } // Get an initial BB to work with from the pool this.curBuf = this.pool.getBuffer(); if (this.curBuf == null) { // No free BB at this moment. Make a new one. The pool returns off heap BBs. Don't make off // heap BB on demand. It is difficult to account for all such and so proper sizing of Max // direct heap size. See HBASE-15525 also for more details. // Make BB with same size of pool's buffer size. this.curBuf = ByteBuffer.allocate(this.pool.getBufferSize()); } else { this.bufsFromPool.add(this.curBuf); } this.allBufs.add(this.curBuf); }
private void initPoolWithAllBuffers(ByteBufferPool pool, int maxBuffersInPool) { ByteBuffer[] buffers = new ByteBuffer[maxBuffersInPool]; // Just call getBuffer() on pool 'maxBuffersInPool' so as to init all buffers and then put back // all. Makes pool with max #buffers. for (int i = 0; i < maxBuffersInPool; i++) { buffers[i] = pool.getBuffer(); } for (ByteBuffer buf : buffers) { pool.putbackBuffer(buf); } } }
@Test public void testAllocateByteBuffToReadInto() throws Exception { int maxBuffersInPool = 10; ByteBufferPool pool = new ByteBufferPool(6 * 1024, maxBuffersInPool); initPoolWithAllBuffers(pool, maxBuffersInPool); ByteBuff buff = null; buff = pair.getFirst(); assertTrue(buff.hasArray()); assertEquals(maxBuffersInPool, pool.getQueueSize()); assertNull(pair.getSecond()); buff = pair.getFirst(); assertFalse(buff.hasArray()); assertEquals(maxBuffersInPool - 1, pool.getQueueSize()); assertNotNull(pair.getSecond()); assertEquals(maxBuffersInPool, pool.getQueueSize()); assertEquals(6 * 1024, bbs[0].limit()); assertEquals(1024, bbs[1].limit()); assertEquals(maxBuffersInPool - 2, pool.getQueueSize()); assertNotNull(pair.getSecond()); assertEquals(maxBuffersInPool, pool.getQueueSize()); assertEquals(6 * 1024, bbs[0].limit()); assertEquals(200, bbs[1].limit()); assertEquals(maxBuffersInPool - 1, pool.getQueueSize()); assertNotNull(pair.getSecond());
/** * Release the resources it uses (The ByteBuffers) which are obtained from pool. Call this only * when all the data is fully used. And it must be called at the end of usage else we will leak * ByteBuffers from pool. */ public void releaseResources() { try { close(); } catch (IOException e) { LOG.debug(e.toString(), e); } // Return back all the BBs to pool if (this.bufsFromPool != null) { for (int i = 0; i < this.bufsFromPool.size(); i++) { this.pool.putbackBuffer(this.bufsFromPool.get(i)); } this.bufsFromPool = null; } this.allBufs = null; this.curBuf = null; }
conf.getInt(HConstants.REGION_SERVER_HANDLER_COUNT, HConstants.DEFAULT_REGION_SERVER_HANDLER_COUNT) * bufsForTwoMB * 2); this.reservoir = new ByteBufferPool(poolBufSize, maxPoolSize); this.minSizeForReservoirUse = getMinSizeForReservoirUse(this.reservoir); } else {
@VisibleForTesting static int getMinSizeForReservoirUse(ByteBufferPool pool) { return pool.getBufferSize() / 6; }
@Test public void testAllocateByteBuffToReadInto() throws Exception { int maxBuffersInPool = 10; ByteBufferPool pool = new ByteBufferPool(6 * 1024, maxBuffersInPool); initPoolWithAllBuffers(pool, maxBuffersInPool); ByteBuff buff = null; buff = pair.getFirst(); assertTrue(buff.hasArray()); assertEquals(maxBuffersInPool, pool.getQueueSize()); assertNull(pair.getSecond()); buff = pair.getFirst(); assertFalse(buff.hasArray()); assertEquals(maxBuffersInPool - 1, pool.getQueueSize()); assertNotNull(pair.getSecond()); assertEquals(maxBuffersInPool, pool.getQueueSize()); assertEquals(6 * 1024, bbs[0].limit()); assertEquals(1024, bbs[1].limit()); assertEquals(maxBuffersInPool - 2, pool.getQueueSize()); assertNotNull(pair.getSecond()); assertEquals(maxBuffersInPool, pool.getQueueSize()); assertEquals(6 * 1024, bbs[0].limit()); assertEquals(200, bbs[1].limit()); assertEquals(maxBuffersInPool - 1, pool.getQueueSize()); assertNotNull(pair.getSecond());
private void initPoolWithAllBuffers(ByteBufferPool pool, int maxBuffersInPool) { ByteBuffer[] buffers = new ByteBuffer[maxBuffersInPool]; // Just call getBuffer() on pool 'maxBuffersInPool' so as to init all buffers and then put back // all. Makes pool with max #buffers. for (int i = 0; i < maxBuffersInPool; i++) { buffers[i] = pool.getBuffer(); } for (ByteBuffer buf : buffers) { pool.putbackBuffer(buf); } } }
private void allocateNewBuffer() { if (this.curBuf != null) { this.curBuf.flip();// On the current buf set limit = pos and pos = 0. } // Get an initial BB to work with from the pool this.curBuf = this.pool.getBuffer(); if (this.curBuf == null) { // No free BB at this moment. Make a new one. The pool returns off heap BBs. Don't make off // heap BB on demand. It is difficult to account for all such and so proper sizing of Max // direct heap size. See HBASE-15525 also for more details. // Make BB with same size of pool's buffer size. this.curBuf = ByteBuffer.allocate(this.pool.getBufferSize()); } else { this.bufsFromPool.add(this.curBuf); } this.allBufs.add(this.curBuf); }
/** * Release the resources it uses (The ByteBuffers) which are obtained from pool. Call this only * when all the data is fully used. And it must be called at the end of usage else we will leak * ByteBuffers from pool. */ public void releaseResources() { try { close(); } catch (IOException e) { LOG.debug(e.toString(), e); } // Return back all the BBs to pool if (this.bufsFromPool != null) { for (int i = 0; i < this.bufsFromPool.size(); i++) { this.pool.putbackBuffer(this.bufsFromPool.get(i)); } this.bufsFromPool = null; } this.allBufs = null; this.curBuf = null; }
@Test public void testWrites() throws Exception { ByteBufferPool pool = new ByteBufferPool(10, 3); ByteBufferListOutputStream bbos = new ByteBufferListOutputStream(pool); bbos.write(2);// Write a byte bbos.write(b); ByteBuffer bb1 = pool.getBuffer(); ByteBuffer bb = ByteBuffer.wrap(Bytes.toBytes("row123_cf1_q1"));// 13 bytes bbos.write(bb, 0, bb.capacity()); pool.putbackBuffer(bb1); bbos.writeInt(123); bbos.writeInt(124); assertEquals(0, pool.getQueueSize()); List<ByteBuffer> allBufs = bbos.getByteBuffers(); assertEquals(4, allBufs.size()); assertEquals(124, b4.getInt()); bbos.releaseResources(); assertEquals(3, pool.getQueueSize());
private void allocateNewBuffer() { if (this.curBuf != null) { this.curBuf.flip();// On the current buf set limit = pos and pos = 0. } // Get an initial BB to work with from the pool this.curBuf = this.pool.getBuffer(); if (this.curBuf == null) { // No free BB at this moment. Make a new one. The pool returns off heap BBs. Don't make off // heap BB on demand. It is difficult to account for all such and so proper sizing of Max // direct heap size. See HBASE-15525 also for more details. // Make BB with same size of pool's buffer size. this.curBuf = ByteBuffer.allocate(this.pool.getBufferSize()); } else { this.bufsFromPool.add(this.curBuf); } this.allBufs.add(this.curBuf); }
/** * Release the resources it uses (The ByteBuffers) which are obtained from pool. Call this only * when all the data is fully used. And it must be called at the end of usage else we will leak * ByteBuffers from pool. */ public void releaseResources() { try { close(); } catch (IOException e) { LOG.debug(e.toString(), e); } // Return back all the BBs to pool if (this.bufsFromPool != null) { for (int i = 0; i < this.bufsFromPool.size(); i++) { this.pool.putbackBuffer(this.bufsFromPool.get(i)); } this.bufsFromPool = null; } this.allBufs = null; this.curBuf = null; }
private void testBBPool(int maxPoolSize, int bufferSize, boolean directByteBuffer) { ByteBufferPool pool = new ByteBufferPool(bufferSize, maxPoolSize, directByteBuffer); for (int i = 0; i < maxPoolSize; i++) { ByteBuffer buffer = pool.getBuffer(); assertEquals(0, buffer.position()); assertEquals(bufferSize, buffer.limit()); assertEquals(directByteBuffer, buffer.isDirect()); } assertEquals(0, pool.getQueueSize()); ByteBuffer bb = directByteBuffer ? ByteBuffer.allocate(bufferSize) : ByteBuffer.allocateDirect(bufferSize); pool.putbackBuffer(bb); assertEquals(0, pool.getQueueSize()); bb = directByteBuffer ? ByteBuffer.allocateDirect(bufferSize + 1) : ByteBuffer.allocate(bufferSize + 1); pool.putbackBuffer(bb); assertEquals(0, pool.getQueueSize()); } }
private void testBBPool(int maxPoolSize, int bufferSize, boolean directByteBuffer) { ByteBufferPool pool = new ByteBufferPool(bufferSize, maxPoolSize, directByteBuffer); for (int i = 0; i < maxPoolSize; i++) { ByteBuffer buffer = pool.getBuffer(); assertEquals(0, buffer.position()); assertEquals(bufferSize, buffer.limit()); assertEquals(directByteBuffer, buffer.isDirect()); } assertEquals(0, pool.getQueueSize()); ByteBuffer bb = directByteBuffer ? ByteBuffer.allocate(bufferSize) : ByteBuffer.allocateDirect(bufferSize); pool.putbackBuffer(bb); assertEquals(0, pool.getQueueSize()); bb = directByteBuffer ? ByteBuffer.allocateDirect(bufferSize + 1) : ByteBuffer.allocate(bufferSize + 1); pool.putbackBuffer(bb); assertEquals(0, pool.getQueueSize()); } }
@Test public void testWrites() throws Exception { ByteBufferPool pool = new ByteBufferPool(10, 3); ByteBufferListOutputStream bbos = new ByteBufferListOutputStream(pool); bbos.write(2);// Write a byte bbos.write(b); ByteBuffer bb1 = pool.getBuffer(); ByteBuffer bb = ByteBuffer.wrap(Bytes.toBytes("row123_cf1_q1"));// 13 bytes bbos.write(bb, 0, bb.capacity()); pool.putbackBuffer(bb1); bbos.writeInt(123); bbos.writeInt(124); assertEquals(0, pool.getQueueSize()); List<ByteBuffer> allBufs = bbos.getByteBuffers(); assertEquals(4, allBufs.size()); assertEquals(124, b4.getInt()); bbos.releaseResources(); assertEquals(3, pool.getQueueSize());
@Test public void testWrites() throws Exception { ByteBufferPool pool = new ByteBufferPool(10, 3); ByteBufferListOutputStream bbos = new ByteBufferListOutputStream(pool); bbos.write(2);// Write a byte bbos.write(b); ByteBuffer bb1 = pool.getBuffer(); ByteBuffer bb = ByteBuffer.wrap(Bytes.toBytes("row123_cf1_q1"));// 13 bytes bbos.write(bb, 0, bb.capacity()); pool.putbackBuffer(bb1); bbos.writeInt(123); bbos.writeInt(124); assertEquals(0, pool.getQueueSize()); List<ByteBuffer> allBufs = bbos.getByteBuffers(); assertEquals(4, allBufs.size()); assertEquals(124, b4.getInt()); bbos.releaseResources(); assertEquals(3, pool.getQueueSize());