@Override public synchronized void setResourceOvercommit() { // Allow the query to use the entire pool. This way the worker will kill the query, if it uses the entire local general pool. // The coordinator will kill the query if the cluster runs out of memory. maxMemory = memoryPool.getMaxBytes(); }
@Override public synchronized void setResourceOvercommit() { // Allow the query to use the entire pool. This way the worker will kill the query, if it uses the entire local general pool. // The coordinator will kill the query if the cluster runs out of memory. maxUserMemory = memoryPool.getMaxBytes(); maxTotalMemory = memoryPool.getMaxBytes(); }
private boolean memoryRevokingNeeded(MemoryPool memoryPool) { return memoryPool.getReservedRevocableBytes() > 0 && memoryPool.getFreeBytes() <= memoryPool.getMaxBytes() * (1.0 - memoryRevokingThreshold); }
private void requestMemoryRevoking(MemoryPool memoryPool, Collection<SqlTask> sqlTasks) { long remainingBytesToRevoke = (long) (-memoryPool.getFreeBytes() + (memoryPool.getMaxBytes() * (1.0 - memoryRevokingTarget))); remainingBytesToRevoke -= getMemoryAlreadyBeingRevoked(sqlTasks, memoryPool); requestRevoking(memoryPool, sqlTasks, remainingBytesToRevoke); }
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); assertEquals(general.getMaxBytes(), general.getFreeBytes()); } } }
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); assertTrue(reserved.isPresent()); reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes()); assertTrue(reserved.get().getFreeBytes() > 0);
List<MemoryPool> memoryPools = server.getLocalMemoryManager().getPools(); assertEquals(memoryPools.size(), 1, "Only general pool should exist"); assertTrue(memoryPools.get(0).tryReserve(fakeQueryId, "test", memoryPools.get(0).getMaxBytes())); assertTrue(general.getReservedBytes() > 0); general.free(fakeQueryId, "test", general.getMaxBytes()); assertTrue(general.getFreeBytes() > 0);
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); assertTrue(reserved.isPresent()); reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes()); assertTrue(reserved.get().getFreeBytes() > 0); Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); general.free(fakeQueryId, "test", general.getMaxBytes()); assertEquals(general.getMaxBytes(), general.getFreeBytes());
long secondQueryMemory = reservedPool.getMaxBytes() - 1; if (useReservedPool) { assertTrue(reservedPool.reserve(secondQuery, "test", secondQueryMemory).isDone());
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); assertEquals(general.getMaxBytes(), general.getFreeBytes()); } } }
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); assertTrue(reserved.isPresent()); reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes()); assertTrue(reserved.get().getFreeBytes() > 0);
List<MemoryPool> memoryPools = server.getLocalMemoryManager().getPools(); assertEquals(memoryPools.size(), 1, "Only general pool should exist"); assertTrue(memoryPools.get(0).tryReserve(fakeQueryId, "test", memoryPools.get(0).getMaxBytes())); assertTrue(general.getReservedBytes() > 0); general.free(fakeQueryId, "test", general.getMaxBytes()); assertTrue(general.getFreeBytes() > 0);
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .put("task.operator-pre-allocated-memory", "0B") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); List<QueryInfo> queryInfos = queryRunner.getCoordinator().getQueryManager().getAllQueryInfo(); for (QueryInfo info : queryInfos) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { MemoryPool reserved = worker.getLocalMemoryManager().getPool(RESERVED_POOL); assertEquals(reserved.getMaxBytes(), reserved.getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getPool(GENERAL_POOL); assertEquals(general.getMaxBytes(), general.getFreeBytes()); MemoryPool system = worker.getLocalMemoryManager().getPool(SYSTEM_POOL); assertEquals(system.getMaxBytes(), system.getFreeBytes()); } } }
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, pool.getMaxBytes())); MemoryPool reserved = server.getLocalMemoryManager().getPool(RESERVED_POOL); reserved.free(fakeQueryId, reserved.getMaxBytes()); assertTrue(reserved.getFreeBytes() > 0);
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); assertTrue(reserved.isPresent()); reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes()); assertTrue(reserved.get().getFreeBytes() > 0); Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); general.free(fakeQueryId, "test", general.getMaxBytes()); assertEquals(general.getMaxBytes(), general.getFreeBytes());
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, pool.getMaxBytes())); MemoryPool reserved = server.getLocalMemoryManager().getPool(RESERVED_POOL); reserved.free(fakeQueryId, reserved.getMaxBytes()); assertTrue(reserved.getFreeBytes() > 0); system.free(fakeQueryId, system.getMaxBytes()); assertTrue(system.getFreeBytes() > 0); assertEquals(reserved.getMaxBytes(), reserved.getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getPool(GENERAL_POOL); general.free(fakeQueryId, general.getMaxBytes()); assertEquals(general.getMaxBytes(), general.getFreeBytes()); MemoryPool system = worker.getLocalMemoryManager().getPool(SYSTEM_POOL); assertEquals(system.getMaxBytes(), system.getFreeBytes());