/** * Test if the waiter that is waiting on availability of more memory is cleaned up when a timeout occurs */ @Test public void testCleanupMemoryAvailabilityWaiterOnBlockTimeout() throws Exception { BufferPool pool = new BufferPool(2, 1, metrics, time, metricGroup); pool.allocate(1, maxBlockTimeMs); try { pool.allocate(2, maxBlockTimeMs); fail("The buffer allocated more memory than its maximum value 2"); } catch (TimeoutException e) { // this is good } assertEquals(0, pool.queued()); assertEquals(1, pool.availableMemory()); }
public void run() { try { for (int i = 0; i < iterations; i++) { int size; if (TestUtils.RANDOM.nextBoolean()) // allocate poolable size size = pool.poolableSize(); else // allocate a random size size = TestUtils.RANDOM.nextInt((int) pool.totalMemory()); ByteBuffer buffer = pool.allocate(size, maxBlockTimeMs); pool.deallocate(buffer); } success.set(true); } catch (Exception e) { e.printStackTrace(); } } }
/** * Allocate a buffer. If buffer allocation fails (e.g. because of OOM) then return the size count back to * available memory and signal the next waiter if it exists. */ private ByteBuffer safeAllocateByteBuffer(int size) { boolean error = true; try { ByteBuffer buffer = allocateByteBuffer(size); error = false; return buffer; } finally { if (error) { this.lock.lock(); try { this.nonPooledAvailableMemory += size; if (!this.waiters.isEmpty()) this.waiters.peekFirst().signal(); } finally { this.lock.unlock(); } } } }
/** * Test that we cannot try to allocate more memory then we have in the whole pool */ @Test(expected = IllegalArgumentException.class) public void testCantAllocateMoreMemoryThanWeHave() throws Exception { BufferPool pool = new BufferPool(1024, 512, metrics, time, metricGroup); ByteBuffer buffer = pool.allocate(1024, maxBlockTimeMs); assertEquals(1024, buffer.limit()); pool.deallocate(buffer); pool.allocate(1025, maxBlockTimeMs); }
int freeListSize = freeSize() * this.poolableSize; if (this.nonPooledAvailableMemory + freeListSize >= size) { freeUp(size); this.nonPooledAvailableMemory -= size; } else { long endWaitNs = time.nanoseconds(); timeNs = Math.max(0L, endWaitNs - startWaitNs); recordWaitTime(timeNs); freeUp(size - accumulated); int got = (int) Math.min(size - accumulated, this.nonPooledAvailableMemory); this.nonPooledAvailableMemory -= got; return safeAllocateByteBuffer(size); else return buffer;
@Test public void testCleanupMemoryAvailabilityOnMetricsException() throws Exception { BufferPool bufferPool = spy(new BufferPool(2, 1, new Metrics(), time, metricGroup)); doThrow(new OutOfMemoryError()).when(bufferPool).recordWaitTime(anyLong()); bufferPool.allocate(1, 0); try { bufferPool.allocate(2, 1000); fail("Expected oom."); } catch (OutOfMemoryError expected) { } assertEquals(1, bufferPool.availableMemory()); assertEquals(0, bufferPool.queued()); assertEquals(1, bufferPool.unallocatedMemory()); //This shouldn't timeout bufferPool.allocate(1, 0); verify(bufferPool).recordWaitTime(anyLong()); }
/** * Test the simple non-blocking allocation paths */ @Test public void testSimple() throws Exception { long totalMemory = 64 * 1024; int size = 1024; BufferPool pool = new BufferPool(totalMemory, size, metrics, time, metricGroup); ByteBuffer buffer = pool.allocate(size, maxBlockTimeMs); assertEquals("Buffer size should equal requested size.", size, buffer.limit()); assertEquals("Unallocated memory should have shrunk", totalMemory - size, pool.unallocatedMemory()); assertEquals("Available memory should have shrunk", totalMemory - size, pool.availableMemory()); buffer.putInt(1); buffer.flip(); pool.deallocate(buffer); assertEquals("All memory should be available", totalMemory, pool.availableMemory()); assertEquals("But now some is on the free list", totalMemory - size, pool.unallocatedMemory()); buffer = pool.allocate(size, maxBlockTimeMs); assertEquals("Recycled buffer should be cleared.", 0, buffer.position()); assertEquals("Recycled buffer should be cleared.", buffer.capacity(), buffer.limit()); pool.deallocate(buffer); assertEquals("All memory should be available", totalMemory, pool.availableMemory()); assertEquals("Still a single buffer on the free list", totalMemory - size, pool.unallocatedMemory()); buffer = pool.allocate(2 * size, maxBlockTimeMs); pool.deallocate(buffer); assertEquals("All memory should be available", totalMemory, pool.availableMemory()); assertEquals("Non-standard size didn't go to the free list.", totalMemory - size, pool.unallocatedMemory()); }
BufferPool pool = new BufferPool(2, 1, metrics, time, metricGroup); long blockTime = 5000; pool.allocate(1, maxBlockTimeMs); Thread t1 = new Thread(new BufferPoolAllocator(pool, blockTime)); Thread t2 = new Thread(new BufferPoolAllocator(pool, blockTime)); Deque<Condition> waiters = pool.waiters(); t2.join(); assertEquals(pool.queued(), 0);
/** * Test if Timeout exception is thrown when there is not enough memory to allocate and the elapsed time is greater than the max specified block time. * And verify that the allocation attempt finishes soon after the maxBlockTimeMs. */ @Test public void testBlockTimeout() throws Exception { BufferPool pool = new BufferPool(10, 1, metrics, Time.SYSTEM, metricGroup); ByteBuffer buffer1 = pool.allocate(1, maxBlockTimeMs); ByteBuffer buffer2 = pool.allocate(1, maxBlockTimeMs); ByteBuffer buffer3 = pool.allocate(1, maxBlockTimeMs); // The first two buffers will be de-allocated within maxBlockTimeMs since the most recent allocation delayedDeallocate(pool, buffer1, maxBlockTimeMs / 2); delayedDeallocate(pool, buffer2, maxBlockTimeMs); // The third buffer will be de-allocated after maxBlockTimeMs since the most recent allocation delayedDeallocate(pool, buffer3, maxBlockTimeMs / 2 * 5); long beginTimeMs = Time.SYSTEM.milliseconds(); try { pool.allocate(10, maxBlockTimeMs); fail("The buffer allocated more memory than its maximum value 10"); } catch (TimeoutException e) { // this is good } // Thread scheduling sometimes means that deallocation varies by this point assertTrue("available memory " + pool.availableMemory(), pool.availableMemory() >= 8 && pool.availableMemory() <= 10); long durationMs = Time.SYSTEM.milliseconds() - beginTimeMs; assertTrue("TimeoutException should not throw before maxBlockTimeMs", durationMs >= maxBlockTimeMs); assertTrue("TimeoutException should throw soon after maxBlockTimeMs", durationMs < maxBlockTimeMs + 1000); }
int size = Math.max(this.batchSize, AbstractRecords.estimateSizeInBytesUpperBound(maxUsableMagic, compression, key, value, headers)); log.trace("Allocating a new {} byte message buffer for topic {} partition {}", size, tp.topic(), tp.partition()); buffer = free.allocate(size, maxTimeToBlock); synchronized (dq) { free.deallocate(buffer); appendsInProgress.decrementAndGet();
public void deallocate(ByteBuffer buffer) { deallocate(buffer, buffer.capacity()); }
@Before public void setup() { Map<String, String> metricTags = new LinkedHashMap<>(); metricTags.put("client-id", CLIENT_ID); int batchSize = 16 * 1024; long deliveryTimeoutMs = 3000L; long totalSize = 1024 * 1024; String metricGrpName = "producer-metrics"; MetricConfig metricConfig = new MetricConfig().tags(metricTags); this.brokerNode = new Node(0, "localhost", 2211); this.transactionManager = new TransactionManager(logContext, transactionalId, transactionTimeoutMs, DEFAULT_RETRY_BACKOFF_MS); Metrics metrics = new Metrics(metricConfig, time); SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(metrics); this.accumulator = new RecordAccumulator(logContext, batchSize, CompressionType.NONE, 0L, 0L, deliveryTimeoutMs, metrics, metricGrpName, time, apiVersions, transactionManager, new BufferPool(totalSize, batchSize, metrics, time, metricGrpName)); this.sender = new Sender(logContext, this.client, this.metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, MAX_RETRIES, senderMetrics, this.time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions); this.client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap("test", 2))); }
/** * Test that delayed allocation blocks */ @Test public void testDelayedAllocation() throws Exception { BufferPool pool = new BufferPool(5 * 1024, 1024, metrics, time, metricGroup); ByteBuffer buffer = pool.allocate(1024, maxBlockTimeMs); CountDownLatch doDealloc = asyncDeallocate(pool, buffer); CountDownLatch allocation = asyncAllocate(pool, 5 * 1024); assertEquals("Allocation shouldn't have happened yet, waiting on memory.", 1L, allocation.getCount()); doDealloc.countDown(); // return the memory assertTrue("Allocation should succeed soon after de-allocation", allocation.await(1, TimeUnit.SECONDS)); }
/** * This test creates lots of threads that hammer on the pool */ @Test public void testStressfulSituation() throws Exception { int numThreads = 10; final int iterations = 50000; final int poolableSize = 1024; final long totalMemory = numThreads / 2 * poolableSize; final BufferPool pool = new BufferPool(totalMemory, poolableSize, metrics, time, metricGroup); List<StressTestThread> threads = new ArrayList<StressTestThread>(); for (int i = 0; i < numThreads; i++) threads.add(new StressTestThread(pool, iterations)); for (StressTestThread thread : threads) thread.start(); for (StressTestThread thread : threads) thread.join(); for (StressTestThread thread : threads) assertTrue("Thread should have completed all iterations successfully.", thread.success.get()); assertEquals(totalMemory, pool.availableMemory()); }
public double measure(MetricConfig config, long now) { return free.queued(); } };
@Test public void outOfMemoryOnAllocation() { BufferPool bufferPool = new BufferPool(1024, 1024, metrics, time, metricGroup) { @Override protected ByteBuffer allocateByteBuffer(int size) { throw new OutOfMemoryError(); } }; try { bufferPool.allocateByteBuffer(1024); // should not reach here fail("Should have thrown OutOfMemoryError"); } catch (OutOfMemoryError ignored) { } assertEquals(bufferPool.availableMemory(), 1024); }
@Override public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedException { ByteBuffer buffer = super.allocate(size, maxTimeToBlockMs); allocatedBuffers.put(buffer, Boolean.TRUE); return buffer; }
public double measure(MetricConfig config, long now) { return free.availableMemory(); } };
public double measure(MetricConfig config, long now) { return free.totalMemory(); } };