@Override public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedException { ByteBuffer buffer = super.allocate(size, maxTimeToBlockMs); allocatedBuffers.put(buffer, Boolean.TRUE); return buffer; }
public void run() { try { pool.allocate(size, maxBlockTimeMs); } catch (InterruptedException e) { e.printStackTrace(); } finally { completed.countDown(); } } };
@Override public void run() { try { pool.allocate(2, maxBlockTimeMs); fail("The buffer allocated more memory than its maximum value 2"); } catch (TimeoutException e) { // this is good } catch (InterruptedException e) { // this can be neglected } } }
/** * Test that we cannot try to allocate more memory then we have in the whole pool */ @Test(expected = IllegalArgumentException.class) public void testCantAllocateMoreMemoryThanWeHave() throws Exception { BufferPool pool = new BufferPool(1024, 512, metrics, time, metricGroup); ByteBuffer buffer = pool.allocate(1024, maxBlockTimeMs); assertEquals(1024, buffer.limit()); pool.deallocate(buffer); pool.allocate(1025, maxBlockTimeMs); }
public void run() { try { for (int i = 0; i < iterations; i++) { int size; if (TestUtils.RANDOM.nextBoolean()) // allocate poolable size size = pool.poolableSize(); else // allocate a random size size = TestUtils.RANDOM.nextInt((int) pool.totalMemory()); ByteBuffer buffer = pool.allocate(size, maxBlockTimeMs); pool.deallocate(buffer); } success.set(true); } catch (Exception e) { e.printStackTrace(); } } }
/** * Test that delayed allocation blocks */ @Test public void testDelayedAllocation() throws Exception { BufferPool pool = new BufferPool(5 * 1024, 1024, metrics, time, metricGroup); ByteBuffer buffer = pool.allocate(1024, maxBlockTimeMs); CountDownLatch doDealloc = asyncDeallocate(pool, buffer); CountDownLatch allocation = asyncAllocate(pool, 5 * 1024); assertEquals("Allocation shouldn't have happened yet, waiting on memory.", 1L, allocation.getCount()); doDealloc.countDown(); // return the memory assertTrue("Allocation should succeed soon after de-allocation", allocation.await(1, TimeUnit.SECONDS)); }
BufferPool pool = new BufferPool(2, 1, metrics, time, metricGroup); long blockTime = 5000; pool.allocate(1, maxBlockTimeMs); Thread t1 = new Thread(new BufferPoolAllocator(pool, blockTime)); Thread t2 = new Thread(new BufferPoolAllocator(pool, blockTime));
@Test public void testCleanupMemoryAvailabilityOnMetricsException() throws Exception { BufferPool bufferPool = spy(new BufferPool(2, 1, new Metrics(), time, metricGroup)); doThrow(new OutOfMemoryError()).when(bufferPool).recordWaitTime(anyLong()); bufferPool.allocate(1, 0); try { bufferPool.allocate(2, 1000); fail("Expected oom."); } catch (OutOfMemoryError expected) { } assertEquals(1, bufferPool.availableMemory()); assertEquals(0, bufferPool.queued()); assertEquals(1, bufferPool.unallocatedMemory()); //This shouldn't timeout bufferPool.allocate(1, 0); verify(bufferPool).recordWaitTime(anyLong()); }
/** * Test if the waiter that is waiting on availability of more memory is cleaned up when a timeout occurs */ @Test public void testCleanupMemoryAvailabilityWaiterOnBlockTimeout() throws Exception { BufferPool pool = new BufferPool(2, 1, metrics, time, metricGroup); pool.allocate(1, maxBlockTimeMs); try { pool.allocate(2, maxBlockTimeMs); fail("The buffer allocated more memory than its maximum value 2"); } catch (TimeoutException e) { // this is good } assertEquals(0, pool.queued()); assertEquals(1, pool.availableMemory()); }
int size = Math.max(this.batchSize, AbstractRecords.estimateSizeInBytesUpperBound(maxUsableMagic, compression, key, value, headers)); log.trace("Allocating a new {} byte message buffer for topic {} partition {}", size, tp.topic(), tp.partition()); buffer = free.allocate(size, maxTimeToBlock); synchronized (dq) {
/** * Test if Timeout exception is thrown when there is not enough memory to allocate and the elapsed time is greater than the max specified block time. * And verify that the allocation attempt finishes soon after the maxBlockTimeMs. */ @Test public void testBlockTimeout() throws Exception { BufferPool pool = new BufferPool(10, 1, metrics, Time.SYSTEM, metricGroup); ByteBuffer buffer1 = pool.allocate(1, maxBlockTimeMs); ByteBuffer buffer2 = pool.allocate(1, maxBlockTimeMs); ByteBuffer buffer3 = pool.allocate(1, maxBlockTimeMs); // The first two buffers will be de-allocated within maxBlockTimeMs since the most recent allocation delayedDeallocate(pool, buffer1, maxBlockTimeMs / 2); delayedDeallocate(pool, buffer2, maxBlockTimeMs); // The third buffer will be de-allocated after maxBlockTimeMs since the most recent allocation delayedDeallocate(pool, buffer3, maxBlockTimeMs / 2 * 5); long beginTimeMs = Time.SYSTEM.milliseconds(); try { pool.allocate(10, maxBlockTimeMs); fail("The buffer allocated more memory than its maximum value 10"); } catch (TimeoutException e) { // this is good } // Thread scheduling sometimes means that deallocation varies by this point assertTrue("available memory " + pool.availableMemory(), pool.availableMemory() >= 8 && pool.availableMemory() <= 10); long durationMs = Time.SYSTEM.milliseconds() - beginTimeMs; assertTrue("TimeoutException should not throw before maxBlockTimeMs", durationMs >= maxBlockTimeMs); assertTrue("TimeoutException should throw soon after maxBlockTimeMs", durationMs < maxBlockTimeMs + 1000); }
/** * Test the simple non-blocking allocation paths */ @Test public void testSimple() throws Exception { long totalMemory = 64 * 1024; int size = 1024; BufferPool pool = new BufferPool(totalMemory, size, metrics, time, metricGroup); ByteBuffer buffer = pool.allocate(size, maxBlockTimeMs); assertEquals("Buffer size should equal requested size.", size, buffer.limit()); assertEquals("Unallocated memory should have shrunk", totalMemory - size, pool.unallocatedMemory()); assertEquals("Available memory should have shrunk", totalMemory - size, pool.availableMemory()); buffer.putInt(1); buffer.flip(); pool.deallocate(buffer); assertEquals("All memory should be available", totalMemory, pool.availableMemory()); assertEquals("But now some is on the free list", totalMemory - size, pool.unallocatedMemory()); buffer = pool.allocate(size, maxBlockTimeMs); assertEquals("Recycled buffer should be cleared.", 0, buffer.position()); assertEquals("Recycled buffer should be cleared.", buffer.capacity(), buffer.limit()); pool.deallocate(buffer); assertEquals("All memory should be available", totalMemory, pool.availableMemory()); assertEquals("Still a single buffer on the free list", totalMemory - size, pool.unallocatedMemory()); buffer = pool.allocate(2 * size, maxBlockTimeMs); pool.deallocate(buffer); assertEquals("All memory should be available", totalMemory, pool.availableMemory()); assertEquals("Non-standard size didn't go to the free list.", totalMemory - size, pool.unallocatedMemory()); }
ByteBuffer buffer = free.allocate(size); synchronized (dq) { RecordBatch last = dq.peekLast();