private void configureLegacyMemoryPools(NodeMemoryConfig config, ReservedSystemMemoryConfig systemMemoryConfig, long availableMemory) { checkArgument(systemMemoryConfig.getReservedSystemMemory().toBytes() < availableMemory, "Reserved memory %s is greater than available heap %s", systemMemoryConfig.getReservedSystemMemory(), new DataSize(availableMemory, BYTE)); maxMemory = new DataSize(availableMemory - systemMemoryConfig.getReservedSystemMemory().toBytes(), BYTE); ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder(); checkArgument(config.getMaxQueryMemoryPerNode().toBytes() <= maxMemory.toBytes(), format("%s set to %s, but only %s of useable heap available", QUERY_MAX_MEMORY_PER_NODE_CONFIG, config.getMaxQueryMemoryPerNode(), maxMemory)); long generalPoolSize = maxMemory.toBytes(); if (config.isReservedPoolEnabled()) { builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryMemoryPerNode())); generalPoolSize -= config.getMaxQueryMemoryPerNode().toBytes(); } builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, new DataSize(generalPoolSize, BYTE))); builder.put(SYSTEM_POOL, new MemoryPool(SYSTEM_POOL, systemMemoryConfig.getReservedSystemMemory())); this.pools = builder.build(); }
public TaskContext build() { MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize); SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(maxSpillSize); DefaultQueryContext queryContext = new DefaultQueryContext( queryId, queryMaxMemory, queryMaxTotalMemory, memoryPool, GC_MONITOR, notificationExecutor, yieldExecutor, queryMaxSpillSize, spillSpaceTracker); return createTaskContext(queryContext, session, taskStateMachine); } }
public TaskContext build() { MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize); SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(maxSpillSize); QueryContext queryContext = new QueryContext( queryId, queryMaxMemory, queryMaxTotalMemory, memoryPool, GC_MONITOR, notificationExecutor, yieldExecutor, queryMaxSpillSize, spillSpaceTracker); return createTaskContext(queryContext, session, taskStateMachine); } }
private void configureMemoryPools(NodeMemoryConfig config, long availableMemory) { validateHeapHeadroom(config, availableMemory); maxMemory = new DataSize(availableMemory - config.getHeapHeadroom().toBytes(), BYTE); checkArgument( config.getMaxQueryMemoryPerNode().toBytes() <= config.getMaxQueryTotalMemoryPerNode().toBytes(), "Max query memory per node (%s) cannot be greater than the max query total memory per node (%s).", QUERY_MAX_MEMORY_PER_NODE_CONFIG, QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG); ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder(); long generalPoolSize = maxMemory.toBytes(); if (config.isReservedPoolEnabled()) { builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryTotalMemoryPerNode())); generalPoolSize -= config.getMaxQueryTotalMemoryPerNode().toBytes(); } verify(generalPoolSize > 0, "general memory pool size is 0"); builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, new DataSize(generalPoolSize, BYTE))); this.pools = builder.build(); }
private void configureMemoryPools(NodeMemoryConfig config, long availableMemory) { validateHeapHeadroom(config, availableMemory); maxMemory = new DataSize(availableMemory - config.getHeapHeadroom().toBytes(), BYTE); checkArgument( config.getMaxQueryMemoryPerNode().toBytes() <= config.getMaxQueryTotalMemoryPerNode().toBytes(), "Max query memory per node (%s) cannot be greater than the max query total memory per node (%s).", QUERY_MAX_MEMORY_PER_NODE_CONFIG, QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG); ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder(); long generalPoolSize = maxMemory.toBytes(); if (config.isReservedPoolEnabled()) { builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryTotalMemoryPerNode())); generalPoolSize -= config.getMaxQueryTotalMemoryPerNode().toBytes(); } verify(generalPoolSize > 0, "general memory pool size is 0"); builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, new DataSize(generalPoolSize, BYTE))); this.pools = builder.build(); }
@Test public void testTaggedAllocations() { QueryId testQuery = new QueryId("test_query"); MemoryPool testPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); testPool.reserve(testQuery, "test_tag", 10); Map<String, Long> allocations = testPool.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); // free 5 bytes for test_tag testPool.free(testQuery, "test_tag", 5); assertEquals(allocations, ImmutableMap.of("test_tag", 5L)); testPool.reserve(testQuery, "test_tag2", 20); assertEquals(allocations, ImmutableMap.of("test_tag", 5L, "test_tag2", 20L)); // free the remaining 5 bytes for test_tag testPool.free(testQuery, "test_tag", 5); assertEquals(allocations, ImmutableMap.of("test_tag2", 20L)); // free all for test_tag2 testPool.free(testQuery, "test_tag2", 20); assertEquals(testPool.getTaggedMemoryAllocations().size(), 0); }
@Test public void testTaggedAllocations() { QueryId testQuery = new QueryId("test_query"); MemoryPool testPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); testPool.reserve(testQuery, "test_tag", 10); Map<String, Long> allocations = testPool.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); // free 5 bytes for test_tag testPool.free(testQuery, "test_tag", 5); assertEquals(allocations, ImmutableMap.of("test_tag", 5L)); testPool.reserve(testQuery, "test_tag2", 20); assertEquals(allocations, ImmutableMap.of("test_tag", 5L, "test_tag2", 20L)); // free the remaining 5 bytes for test_tag testPool.free(testQuery, "test_tag", 5); assertEquals(allocations, ImmutableMap.of("test_tag2", 20L)); // free all for test_tag2 testPool.free(testQuery, "test_tag2", 20); assertEquals(testPool.getTaggedMemoryAllocations().size(), 0); }
@Test public void testMoveQuery() { QueryId testQuery = new QueryId("test_query"); MemoryPool pool1 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); MemoryPool pool2 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); pool1.reserve(testQuery, "test_tag", 10); Map<String, Long> allocations = pool1.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); pool1.moveQuery(testQuery, pool2); assertNull(pool1.getTaggedMemoryAllocations().get(testQuery)); allocations = pool2.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); assertEquals(pool1.getFreeBytes(), 1000); assertEquals(pool2.getFreeBytes(), 990); pool2.free(testQuery, "test", 10); assertEquals(pool2.getFreeBytes(), 1000); }
@Test public void testMoveQuery() { QueryId testQuery = new QueryId("test_query"); MemoryPool pool1 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); MemoryPool pool2 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); pool1.reserve(testQuery, "test_tag", 10); Map<String, Long> allocations = pool1.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); pool1.moveQuery(testQuery, pool2); assertNull(pool1.getTaggedMemoryAllocations().get(testQuery)); allocations = pool2.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); assertEquals(pool1.getFreeBytes(), 1000); assertEquals(pool2.getFreeBytes(), 990); pool2.free(testQuery, "test", 10); assertEquals(pool2.getFreeBytes(), 1000); }
@BeforeMethod public void setUp() { memoryPool = new MemoryPool(GENERAL_POOL, new DataSize(10, BYTE)); TaskExecutor taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker()); taskExecutor.start(); // Must be single threaded executor = newScheduledThreadPool(1, threadsNamed("task-notification-%s")); scheduledExecutor = newScheduledThreadPool(2, threadsNamed("task-notification-%s")); LocalExecutionPlanner planner = createTestingPlanner(); sqlTaskExecutionFactory = new SqlTaskExecutionFactory( executor, taskExecutor, planner, createTestSplitMonitor(), new TaskManagerConfig()); allOperatorContexts = null; }
@BeforeMethod public void setUp() { memoryPool = new MemoryPool(GENERAL_POOL, new DataSize(10, BYTE)); TaskExecutor taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker()); taskExecutor.start(); // Must be single threaded executor = newScheduledThreadPool(1, threadsNamed("task-notification-%s")); scheduledExecutor = newScheduledThreadPool(2, threadsNamed("task-notification-%s")); LocalExecutionPlanner planner = createTestingPlanner(); sqlTaskExecutionFactory = new SqlTaskExecutionFactory( executor, taskExecutor, planner, createTestSplitMonitor(), new TaskManagerConfig()); allOperatorContexts = null; }
private TaskContext newTestingTaskContext(ScheduledExecutorService taskNotificationExecutor, ScheduledExecutorService driverYieldExecutor, TaskStateMachine taskStateMachine) { DefaultQueryContext queryContext = new DefaultQueryContext( new QueryId("queryid"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE))); return queryContext.addTaskContext(taskStateMachine, TEST_SESSION, false, false, OptionalInt.empty()); }
private TaskContext newTestingTaskContext(ScheduledExecutorService taskNotificationExecutor, ScheduledExecutorService driverYieldExecutor, TaskStateMachine taskStateMachine) { QueryContext queryContext = new QueryContext( new QueryId("queryid"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE))); return queryContext.addTaskContext(taskStateMachine, TEST_SESSION, false, false, OptionalInt.empty()); }
@Test public void testCountAlreadyRevokedMemoryWithinAPool() throws Exception { // Given SqlTask sqlTask1 = newSqlTask(); MemoryPool anotherMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(10, BYTE)); sqlTask1.getQueryContext().setMemoryPool(anotherMemoryPool); OperatorContext operatorContext1 = createContexts(sqlTask1); SqlTask sqlTask2 = newSqlTask(); OperatorContext operatorContext2 = createContexts(sqlTask2); List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2); MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(asList(memoryPool, anotherMemoryPool), () -> tasks, executor, 1.0, 1.0); allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2); /* * sqlTask1 fills its pool */ operatorContext1.localRevocableMemoryContext().setBytes(12); requestMemoryRevoking(scheduler); assertMemoryRevokingRequestedFor(operatorContext1); /* * When sqlTask2 fills its pool */ operatorContext2.localRevocableMemoryContext().setBytes(12); requestMemoryRevoking(scheduler); /* * Then sqlTask2 should be asked to revoke its memory too */ assertMemoryRevokingRequestedFor(operatorContext1, operatorContext2); }
@BeforeMethod public void setUpTest() { memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize); queryContext = new DefaultQueryContext( new QueryId("test_query"), queryMaxMemory, queryMaxTotalMemory, memoryPool, new TestingGcMonitor(), notificationExecutor, yieldExecutor, queryMaxSpillSize, spillSpaceTracker); taskContext = queryContext.addTaskContext( new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor), testSessionBuilder().build(), true, true, OptionalInt.empty()); pipelineContext = taskContext.addPipelineContext(0, true, true, false); driverContext = pipelineContext.addDriverContext(); operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("a"), "test-operator"); }
public SqlTask createInitialTask() { TaskId taskId = new TaskId("query", 0, nextTaskId.incrementAndGet()); URI location = URI.create("fake://task/" + taskId); QueryContext queryContext = new QueryContext(new QueryId("query"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE))); queryContext.addTaskContext(new TaskStateMachine(taskId, taskNotificationExecutor), testSessionBuilder().build(), false, false, OptionalInt.empty()); return createSqlTask( taskId, location, "fake", queryContext, sqlTaskExecutionFactory, taskNotificationExecutor, Functions.identity(), new DataSize(32, MEGABYTE), new CounterStat()); } }
@BeforeMethod public void setUpTest() { memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize); queryContext = new QueryContext( new QueryId("test_query"), queryMaxMemory, queryMaxTotalMemory, memoryPool, new TestingGcMonitor(), notificationExecutor, yieldExecutor, queryMaxSpillSize, spillSpaceTracker); taskContext = queryContext.addTaskContext( new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor), testSessionBuilder().build(), true, true, OptionalInt.empty()); pipelineContext = taskContext.addPipelineContext(0, true, true, false); driverContext = pipelineContext.addDriverContext(); operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("a"), "test-operator"); }
public SqlTask createInitialTask() { TaskId taskId = new TaskId("query", 0, nextTaskId.incrementAndGet()); URI location = URI.create("fake://task/" + taskId); DefaultQueryContext queryContext = new DefaultQueryContext(new QueryId("query"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE))); queryContext.addTaskContext(new TaskStateMachine(taskId, taskNotificationExecutor), testSessionBuilder().build(), false, false, OptionalInt.empty()); return createSqlTask( taskId, location, "fake", queryContext, sqlTaskExecutionFactory, taskNotificationExecutor, Functions.identity(), new DataSize(32, MEGABYTE), new CounterStat()); } }
private void setUp(Supplier<List<Driver>> driversSupplier) { checkState(localQueryRunner == null, "Already set up"); Session session = testSessionBuilder() .setCatalog("tpch") .setSchema("tiny") .setSystemProperty("task_default_concurrency", "1") .build(); localQueryRunner = queryRunnerWithInitialTransaction(session); // add tpch localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of()); userPool = new MemoryPool(new MemoryPoolId("test"), TEN_MEGABYTES); fakeQueryId = new QueryId("fake"); SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE)); QueryContext queryContext = new QueryContext(new QueryId("query"), TEN_MEGABYTES, new DataSize(20, MEGABYTE), userPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), TEN_MEGABYTES, spillSpaceTracker); taskContext = createTaskContext(queryContext, localQueryRunner.getExecutor(), session); drivers = driversSupplier.get(); }
private void setUp(Supplier<List<Driver>> driversSupplier) { checkState(localQueryRunner == null, "Already set up"); Session session = testSessionBuilder() .setCatalog("tpch") .setSchema("tiny") .setSystemProperty("task_default_concurrency", "1") .build(); localQueryRunner = queryRunnerWithInitialTransaction(session); // add tpch localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of()); userPool = new MemoryPool(new MemoryPoolId("test"), TEN_MEGABYTES); fakeQueryId = new QueryId("fake"); SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE)); DefaultQueryContext queryContext = new DefaultQueryContext(new QueryId("query"), TEN_MEGABYTES, new DataSize(20, MEGABYTE), userPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), TEN_MEGABYTES, spillSpaceTracker); taskContext = createTaskContext(queryContext, localQueryRunner.getExecutor(), session); drivers = driversSupplier.get(); }