@Test(expectedExceptions = UnsupportedOperationException.class) public void testCoordinatorShutdown() throws Exception { try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, ImmutableMap.of())) { TestingPrestoServer coordinator = queryRunner.getServers() .stream() .filter(TestingPrestoServer::isCoordinator) .findFirst() .get(); coordinator.getGracefulShutdownHandler().requestShutdown(); } } }
private void closeWorker() throws Exception { int nodeCount = getNodeCount(); DistributedQueryRunner queryRunner = (DistributedQueryRunner) getQueryRunner(); TestingPrestoServer worker = queryRunner.getServers().stream() .filter(server -> !server.isCoordinator()) .findAny() .orElseThrow(() -> new IllegalStateException("No worker nodes")); worker.close(); waitForNodes(nodeCount - 1); }
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); for (TestingPrestoServer server : queryRunner.getServers()) { Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent());
@Test public void testInsufficientWorkerNodesAfterDrop() throws Exception { try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder() .setCoordinatorProperties(ImmutableMap.<String, String>builder() .put("query-manager.required-workers", "4") .put("query-manager.required-workers-max-wait", "1ns") .build()) .setNodeCount(4) .build()) { queryRunner.execute("SELECT 1"); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 4); try { // Query should still be allowed to run if active workers drop down below the minimum required nodes queryRunner.getServers().get(0).close(); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 3); queryRunner.execute("SELECT 1"); } catch (RuntimeException e) { assertEquals(e.getMessage(), "Insufficient active worker nodes. Waited 1.00ns for at least 4 workers, but only 3 workers are active"); } } } }
TestingPrestoServer worker = queryRunner.getServers() .stream() .filter(server -> !server.isCoordinator())
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); assertEquals(general.getMaxBytes(), general.getFreeBytes()); } } }
for (TestingPrestoServer server : queryRunner.getServers()) { List<MemoryPool> memoryPools = server.getLocalMemoryManager().getPools(); assertEquals(memoryPools.size(), 1, "Only general pool should exist"); for (TestingPrestoServer server : queryRunner.getServers()) { Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); MemoryPool general = server.getLocalMemoryManager().getGeneralPool();
@Test public void testSufficientInitialWorkerNodes() throws Exception { try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder() .setSingleCoordinatorProperty("query-manager.initialization-required-workers", "4") .setNodeCount(4) .build()) { queryRunner.execute("SELECT 1"); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 4); // Query should still be allowed to run if active workers drop down below the minimum required nodes queryRunner.getServers().get(0).close(); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 3); queryRunner.execute("SELECT 1"); } }
@Test(timeOut = 60_000L) public void testLegacyQueryContext() throws Exception { QueryManager queryManager = queryRunner.getCoordinator().getQueryManager(); QueryId queryId = queryManager.createQueryId(); queryManager.createQuery( queryId, new TestingSessionContext(TEST_SESSION), "SELECT * FROM lineitem") .get(); waitForQueryState(queryRunner, queryId, RUNNING); // cancel query queryManager.failQuery(queryId, new PrestoException(GENERIC_INTERNAL_ERROR, "mock exception")); // assert that LegacyQueryContext is used instead of the DefaultQueryContext SqlTaskManager taskManager = (SqlTaskManager) queryRunner.getServers().get(0).getTaskManager(); assertInstanceOf(taskManager.getQueryContext(queryId), LegacyQueryContext.class); } }
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); for (TestingPrestoServer server : queryRunner.getServers()) { Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); for (TestingPrestoServer worker : queryRunner.getServers()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent());
@Test(expectedExceptions = UnsupportedOperationException.class) public void testCoordinatorShutdown() throws Exception { try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, ImmutableMap.of())) { TestingPrestoServer coordinator = queryRunner.getServers() .stream() .filter(TestingPrestoServer::isCoordinator) .findFirst() .get(); coordinator.getGracefulShutdownHandler().requestShutdown(); } } }
@Test(expectedExceptions = UnsupportedOperationException.class) public void testCoordinatorShutdown() throws Exception { try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, ImmutableMap.of())) { TestingPrestoServer coordinator = queryRunner.getServers() .stream() .filter(TestingPrestoServer::isCoordinator) .findFirst() .get(); coordinator.getGracefulShutdownHandler().requestShutdown(); } } }
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); for (TestingPrestoServer server : queryRunner.getServers()) { Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent());
for (TestingPrestoServer server : queryRunner.getServers()) { List<MemoryPool> memoryPools = server.getLocalMemoryManager().getPools(); assertEquals(memoryPools.size(), 1, "Only general pool should exist"); for (TestingPrestoServer server : queryRunner.getServers()) { Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); MemoryPool general = server.getLocalMemoryManager().getGeneralPool();
TestingPrestoServer worker = queryRunner.getServers() .stream() .filter(server -> !server.isCoordinator())
@Test public void testInsufficientWorkerNodesAfterDrop() throws Exception { try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder() .setCoordinatorProperties(ImmutableMap.<String, String>builder() .put("query-manager.required-workers", "4") .put("query-manager.required-workers-max-wait", "1ns") .build()) .setNodeCount(4) .build()) { queryRunner.execute("SELECT 1"); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 4); try { // Query should still be allowed to run if active workers drop down below the minimum required nodes queryRunner.getServers().get(0).close(); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 3); queryRunner.execute("SELECT 1"); } catch (RuntimeException e) { assertEquals(e.getMessage(), "Insufficient active worker nodes. Waited 1.00ns for at least 4 workers, but only 3 workers are active"); } } } }
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); assertEquals(general.getMaxBytes(), general.getFreeBytes()); } } }
@Test public void testSufficientInitialWorkerNodes() throws Exception { try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder() .setSingleCoordinatorProperty("query-manager.initialization-required-workers", "4") .setNodeCount(4) .build()) { queryRunner.execute("SELECT 1"); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 4); // Query should still be allowed to run if active workers drop down below the minimum required nodes queryRunner.getServers().get(0).close(); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 3); queryRunner.execute("SELECT 1"); } }
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .put("task.operator-pre-allocated-memory", "0B") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); List<QueryInfo> queryInfos = queryRunner.getCoordinator().getQueryManager().getAllQueryInfo(); for (QueryInfo info : queryInfos) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { MemoryPool reserved = worker.getLocalMemoryManager().getPool(RESERVED_POOL); assertEquals(reserved.getMaxBytes(), reserved.getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getPool(GENERAL_POOL); assertEquals(general.getMaxBytes(), general.getFreeBytes()); MemoryPool system = worker.getLocalMemoryManager().getPool(SYSTEM_POOL); assertEquals(system.getMaxBytes(), system.getFreeBytes()); } } }
@Test(timeOut = 60_000L) public void testLegacyQueryContext() throws Exception { QueryManager queryManager = queryRunner.getCoordinator().getQueryManager(); QueryId queryId = queryManager.createQueryId(); queryManager.createQuery( queryId, new TestingSessionContext(TEST_SESSION), "SELECT * FROM lineitem") .get(); waitForQueryState(queryRunner, queryId, RUNNING); // cancel query queryManager.failQuery(queryId, new PrestoException(GENERIC_INTERNAL_ERROR, "mock exception")); // assert that LegacyQueryContext is used instead of the DefaultQueryContext SqlTaskManager taskManager = (SqlTaskManager) queryRunner.getServers().get(0).getTaskManager(); assertInstanceOf(taskManager.getQueryContext(queryId), LegacyQueryContext.class); } }