@Inject public LocalMemoryManagerExporter(LocalMemoryManager memoryManager, MBeanExporter exporter) { this.exporter = requireNonNull(exporter, "exporter is null"); for (MemoryPool pool : memoryManager.getPools()) { addPool(pool); } }
private static List<MemoryPool> getMemoryPools(LocalMemoryManager localMemoryManager) { requireNonNull(localMemoryManager, "localMemoryManager can not be null"); ImmutableList.Builder<MemoryPool> builder = new ImmutableList.Builder<>(); builder.add(localMemoryManager.getGeneralPool()); localMemoryManager.getReservedPool().ifPresent(builder::add); return builder.build(); }
@POST @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public MemoryInfo getMemoryInfo(MemoryPoolAssignmentsRequest request) { taskManager.updateMemoryPoolAssignments(request); return memoryManager.getInfo(); }
@GET @Path("{poolId}") public Response getMemoryInfo(@PathParam("poolId") String poolId) { if (GENERAL_POOL.getId().equals(poolId)) { return toSuccessfulResponse(memoryManager.getGeneralPool().getInfo()); } if (SYSTEM_POOL.getId().equals(poolId) && memoryManager.getSystemPool().isPresent()) { return toSuccessfulResponse(memoryManager.getSystemPool().get().getInfo()); } if (RESERVED_POOL.getId().equals(poolId) && memoryManager.getReservedPool().isPresent()) { return toSuccessfulResponse(memoryManager.getReservedPool().get().getInfo()); } return Response.status(NOT_FOUND).build(); }
Optional<MemoryPool> systemPool = localMemoryManager.getSystemPool(); verify(systemPool.isPresent(), "systemPool must be present"); return new LegacyQueryContext( queryId, maxQueryUserMemoryPerNode, localMemoryManager.getGeneralPool(), systemPool.get(), gcMonitor, maxQueryUserMemoryPerNode, maxQueryTotalMemoryPerNode, localMemoryManager.getGeneralPool(), gcMonitor, taskNotificationExecutor,
List<MemoryPool> memoryPools = server.getLocalMemoryManager().getPools(); assertEquals(memoryPools.size(), 1, "Only general pool should exist"); assertTrue(memoryPools.get(0).tryReserve(fakeQueryId, "test", memoryPools.get(0).getMaxBytes())); Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); MemoryPool general = server.getLocalMemoryManager().getGeneralPool(); assertFalse(reserved.isPresent()); assertTrue(general.getReservedBytes() > 0);
for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent());
private void configureMemoryPools(NodeMemoryConfig config, long availableMemory) { validateHeapHeadroom(config, availableMemory); maxMemory = new DataSize(availableMemory - config.getHeapHeadroom().toBytes(), BYTE); checkArgument( config.getMaxQueryMemoryPerNode().toBytes() <= config.getMaxQueryTotalMemoryPerNode().toBytes(), "Max query memory per node (%s) cannot be greater than the max query total memory per node (%s).", QUERY_MAX_MEMORY_PER_NODE_CONFIG, QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG); ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder(); long generalPoolSize = maxMemory.toBytes(); if (config.isReservedPoolEnabled()) { builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryTotalMemoryPerNode())); generalPoolSize -= config.getMaxQueryTotalMemoryPerNode().toBytes(); } verify(generalPoolSize > 0, "general memory pool size is 0"); builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, new DataSize(generalPoolSize, BYTE))); this.pools = builder.build(); }
public TestSqlTaskManager() { localMemoryManager = new LocalMemoryManager(new NodeMemoryConfig(), new ReservedSystemMemoryConfig()); localSpillManager = new LocalSpillManager(new NodeSpillConfig()); taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker()); taskExecutor.start(); taskManagementExecutor = new TaskManagementExecutor(); }
for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool();
for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent());
for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, pool.getMaxBytes())); MemoryPool reserved = server.getLocalMemoryManager().getPool(RESERVED_POOL); MemoryPool system = server.getLocalMemoryManager().getPool(SYSTEM_POOL); MemoryPool reserved = worker.getLocalMemoryManager().getPool(RESERVED_POOL); assertEquals(reserved.getMaxBytes(), reserved.getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getPool(GENERAL_POOL); MemoryPool system = worker.getLocalMemoryManager().getPool(SYSTEM_POOL); assertEquals(system.getMaxBytes(), system.getFreeBytes());
@Override public synchronized void updateMemoryPoolAssignments(MemoryPoolAssignmentsRequest assignments) { if (coordinatorId != null && coordinatorId.equals(assignments.getCoordinatorId()) && assignments.getVersion() <= currentMemoryPoolAssignmentVersion) { return; } currentMemoryPoolAssignmentVersion = assignments.getVersion(); if (coordinatorId != null && !coordinatorId.equals(assignments.getCoordinatorId())) { log.warn("Switching coordinator affinity from " + coordinatorId + " to " + assignments.getCoordinatorId()); } coordinatorId = assignments.getCoordinatorId(); for (MemoryPoolAssignment assignment : assignments.getAssignments()) { queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(localMemoryManager.getPool(assignment.getPoolId())); } }
@Test(expectedExceptions = IllegalArgumentException.class) public void testInvalidValues() { NodeMemoryConfig config = new NodeMemoryConfig(); config.setMaxQueryTotalMemoryPerNode(new DataSize(1, GIGABYTE)); config.setHeapHeadroom(new DataSize(3.1, GIGABYTE)); // In this case we have 4GB - 1GB = 3GB available memory for the general pool // and the heap headroom and the config is more than that. validateHeapHeadroom(config, new DataSize(4, GIGABYTE).toBytes()); } }
public TestSqlTaskManager() { localMemoryManager = new LocalMemoryManager(new MemoryManagerConfig(), new ReservedSystemMemoryConfig()); taskExecutor = new TaskExecutor(8, 16); taskExecutor.start(); }
List<MemoryPool> memoryPools = server.getLocalMemoryManager().getPools(); assertEquals(memoryPools.size(), 1, "Only general pool should exist"); assertTrue(memoryPools.get(0).tryReserve(fakeQueryId, "test", memoryPools.get(0).getMaxBytes())); Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); MemoryPool general = server.getLocalMemoryManager().getGeneralPool(); assertFalse(reserved.isPresent()); assertTrue(general.getReservedBytes() > 0);
@Override public synchronized void updateMemoryPoolAssignments(MemoryPoolAssignmentsRequest assignments) { if (coordinatorId != null && coordinatorId.equals(assignments.getCoordinatorId()) && assignments.getVersion() <= currentMemoryPoolAssignmentVersion) { return; } currentMemoryPoolAssignmentVersion = assignments.getVersion(); if (coordinatorId != null && !coordinatorId.equals(assignments.getCoordinatorId())) { log.warn("Switching coordinator affinity from " + coordinatorId + " to " + assignments.getCoordinatorId()); } coordinatorId = assignments.getCoordinatorId(); for (MemoryPoolAssignment assignment : assignments.getAssignments()) { if (assignment.getPoolId().equals(GENERAL_POOL)) { queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(localMemoryManager.getGeneralPool()); } else if (assignment.getPoolId().equals(RESERVED_POOL)) { MemoryPool reservedPool = localMemoryManager.getReservedPool() .orElseThrow(() -> new IllegalArgumentException(format("Cannot move %s to the reserved pool as the reserved pool is not enabled", assignment.getQueryId()))); queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(reservedPool); } else { new IllegalArgumentException(format("Cannot move %s to %s as the target memory pool id is invalid", assignment.getQueryId(), assignment.getPoolId())); } } }
@Inject public LocalMemoryManagerExporter(LocalMemoryManager memoryManager, MBeanExporter exporter) { this.exporter = requireNonNull(exporter, "exporter is null"); for (MemoryPool pool : memoryManager.getPools()) { addPool(pool); } }