public HostAndPort getAddress() { return HostAndPort.fromParts(getBaseUrl().getHost(), getBaseUrl().getPort()); }
public void run() throws Exception { try (TestingPrestoServer server = new TestingPrestoServer()) { for (String pluginClass : options.getPluginClassNames()) { Plugin plugin = (Plugin) Class.forName(pluginClass).getConstructor().newInstance(); server.installPlugin(plugin); } for (Catalog catalog : options.getCatalogs()) { server.createCatalog(catalog.getCatalogName(), catalog.getConnectorName()); } System.out.println(server.getAddress()); waitForInterruption(); } }
@Override public Metadata getMetadata() { return server.getMetadata(); }
private boolean isConnectionVisibleToAllNodes(ConnectorId connectorId) { for (TestingPrestoServer server : servers) { server.refreshNodes(); Set<Node> activeNodesWithConnector = server.getActiveNodesWithConnector(connectorId); if (activeNodesWithConnector.size() != servers.size()) { return false; } } return true; }
public boolean tableExists(Session session, String table) { return transaction(prestoServer.getTransactionManager(), prestoServer.getAccessControl()) .readOnly() .execute(session, transactionSession -> { return MetadataUtil.tableExists(prestoServer.getMetadata(), transactionSession, table); }); }
private void assertResourceGroup(DistributedQueryRunner queryRunner, Session session, String query, ResourceGroupId expectedResourceGroup) throws InterruptedException { QueryId queryId = createQuery(queryRunner, session, query); waitForQueryState(queryRunner, queryId, ImmutableSet.of(RUNNING, FINISHED)); Optional<ResourceGroupId> resourceGroupId = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(queryId).getResourceGroupId(); assertTrue(resourceGroupId.isPresent(), "Query should have a resource group"); assertEquals(resourceGroupId.get(), expectedResourceGroup, format("Expected: '%s' resource group, found: %s", expectedResourceGroup, resourceGroupId.get())); }
private TableMetadata getTableMetadata(String catalog, String schema, String tableName) { Session session = getSession(); Metadata metadata = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getMetadata(); return transaction(getQueryRunner().getTransactionManager(), getQueryRunner().getAccessControl()) .readOnly() .execute(session, transactionSession -> { Optional<TableHandle> tableHandle = metadata.getTableHandle(transactionSession, new QualifiedObjectName(catalog, schema, tableName)); assertTrue(tableHandle.isPresent()); return metadata.getTableMetadata(transactionSession, tableHandle.get()); }); }
ImmutableMap.Builder<String, String> serverProperties = ImmutableMap.<String, String>builder() .putAll(properties) .put("coordinator", String.valueOf(coordinator)) .put("presto.version", "testversion") .put("task.concurrency", "4") .put("task.max-worker-threads", "4") .add(new TestingNodeModule(Optional.ofNullable(environment))) .add(new TestingHttpServerModule(parseInt(coordinator ? coordinatorPort : "0"))) .add(new JsonModule()) pageSourceManager = injector.getInstance(PageSourceManager.class); if (coordinator) { resourceGroupManager = Optional.of(injector.getInstance(InternalResourceGroupManager.class)); nodePartitioningManager = injector.getInstance(NodePartitioningManager.class); clusterMemoryManager = injector.getInstance(ClusterMemoryManager.class); resourceGroupManager = Optional.empty(); nodePartitioningManager = null; clusterMemoryManager = null; refreshNodes();
for (TestingPrestoServer server : queryRunner.getServers()) { for (MemoryPool pool : server.getLocalMemoryManager().getPools()) { assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes())); queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk"))); ClusterMemoryManager memoryManager = queryRunner.getCoordinator().getClusterMemoryManager(); ClusterMemoryPool reservedPool; while ((reservedPool = memoryManager.getPools().get(RESERVED_POOL)) == null) { assertNotNull(generalPool); List<BasicQueryInfo> currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries(); for (BasicQueryInfo info : currentQueryInfos) { assertFalse(info.getState().isDone()); currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries(); Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool();
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); assertEquals(general.getMaxBytes(), general.getFreeBytes()); } } }
private void testRejection() throws Exception { try (DistributedQueryRunner queryRunner = createQueryRunner()) { queryRunner.installPlugin(new ResourceGroupManagerPlugin()); queryRunner.getCoordinator().getResourceGroupManager().get().setConfigurationManager("file", ImmutableMap.of("resource-groups.config-file", getResourceFilePath("resource_groups_config_dashboard.json"))); QueryId queryId = createQuery(queryRunner, newRejectionSession(), LONG_LASTING_QUERY); waitForQueryState(queryRunner, queryId, FAILED); QueryManager queryManager = queryRunner.getCoordinator().getQueryManager(); assertEquals(queryManager.getQueryInfo(queryId).getErrorCode(), QUERY_REJECTED.toErrorCode()); } }
@Test(timeOut = SHUTDOWN_TIMEOUT_MILLIS) public void testShutdown() throws Exception queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk"))); TestingPrestoServer worker = queryRunner.getServers() .stream() .filter(server -> !server.isCoordinator()) .findFirst() .get(); TaskManager taskManager = worker.getTaskManager(); worker.getGracefulShutdownHandler().requestShutdown(); List<BasicQueryInfo> queryInfos = queryRunner.getCoordinator().getQueryManager().getQueries(); for (BasicQueryInfo info : queryInfos) { assertEquals(info.getState(), FINISHED); TestShutdownAction shutdownAction = (TestShutdownAction) worker.getShutdownAction(); shutdownAction.waitForShutdownComplete(SHUTDOWN_TIMEOUT_MILLIS); assertTrue(shutdownAction.isWorkerShutdown());
@Test public void testTopicExists() { QualifiedObjectName name = new QualifiedObjectName("kafka", "default", topicName); transaction(queryRunner.getTransactionManager(), new AllowAllAccessControl()) .singleStatement() .execute(SESSION, session -> { Optional<TableHandle> handle = queryRunner.getServer().getMetadata().getTableHandle(session, name); assertTrue(handle.isPresent()); }); }
@Test public void testQuery() assertNull(queryResults.getError()); BasicQueryInfo queryInfo = server.getQueryManager().getQueryInfo(new QueryId(queryResults.getId())); assertEquals(queryInfo.getSession().getSystemProperties(), ImmutableMap.builder() .put(QUERY_MAX_MEMORY, "1GB") .put(JOIN_DISTRIBUTION_TYPE, "partitioned") .put(HASH_PARTITION_COUNT, "43") .build()); assertEquals(queryInfo.getSession().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}"); assertEquals(queryInfo.getSession().getPreparedStatements(), ImmutableMap.builder() .put("foo", "select * from bar") .build());
@Test public void testNonLeafGroup() throws Exception { Session session = testSessionBuilder() .setCatalog("tpch") .setSchema("sf100000") .setSource("non-leaf") .build(); QueryManager queryManager = queryRunner.getCoordinator().getQueryManager(); InternalResourceGroupManager manager = queryRunner.getCoordinator().getResourceGroupManager().get(); DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager(); int originalSize = getSelectors(queryRunner).size(); // Add a selector for a non leaf group dao.insertSelector(3, 100, "user.*", "(?i).*non-leaf.*", null, null, null); dbConfigurationManager.load(); while (getSelectors(queryRunner).size() != originalSize + 1) { MILLISECONDS.sleep(500); } // Submit query with side effect of creating resource groups QueryId firstDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY); waitForQueryState(queryRunner, firstDashboardQuery, RUNNING); cancelQuery(queryRunner, firstDashboardQuery); waitForQueryState(queryRunner, firstDashboardQuery, FAILED); // Submit a query to a non-leaf resource group QueryId invalidResourceGroupQuery = createQuery(queryRunner, session, LONG_LASTING_QUERY); waitForQueryState(queryRunner, invalidResourceGroupQuery, FAILED); assertEquals(queryRunner.getQueryInfo(invalidResourceGroupQuery).getErrorCode(), INVALID_RESOURCE_GROUP.toErrorCode()); }
@Test public void testSufficientInitialWorkerNodes() throws Exception { try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder() .setSingleCoordinatorProperty("query-manager.initialization-required-workers", "4") .setNodeCount(4) .build()) { queryRunner.execute("SELECT 1"); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 4); // Query should still be allowed to run if active workers drop down below the minimum required nodes queryRunner.getServers().get(0).close(); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 3); queryRunner.execute("SELECT 1"); } }
@BeforeClass public void setup() throws Exception { Logging.initialize(); URL resource = getClass().getClassLoader().getResource("33.privateKey"); assertNotNull(resource, "key directory not found"); File keyDir = new File(resource.getFile()).getAbsoluteFile().getParentFile(); defaultKey = getMimeDecoder().decode(asCharSource(new File(keyDir, "default-key.key"), US_ASCII).read().getBytes(US_ASCII)); hmac222 = getMimeDecoder().decode(asCharSource(new File(keyDir, "222.key"), US_ASCII).read().getBytes(US_ASCII)); privateKey33 = PemReader.loadPrivateKey(new File(keyDir, "33.privateKey"), Optional.empty()); server = new TestingPrestoServer( true, ImmutableMap.<String, String>builder() .put("http-server.authentication.type", "JWT") .put("http.authentication.jwt.key-file", new File(keyDir, "${KID}.key").toString()) .put("http-server.https.enabled", "true") .put("http-server.https.keystore.path", getResource("localhost.keystore").getPath()) .put("http-server.https.keystore.key", "changeit") .build(), null, null, new SqlParserOptions(), ImmutableList.of()); server.installPlugin(new TpchPlugin()); server.createCatalog(TEST_CATALOG, "tpch"); waitForNodeRefresh(server); }
try { ClientSession clientSession = new ClientSession( queryRunner.getCoordinator().getBaseUrl(), "user", "source", Optional.empty(), ImmutableSet.of(), null, "America/Los_Angeles", Locale.ENGLISH, ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), null, new Duration(2, MINUTES));
private HiveInsertTableHandle getHiveInsertTableHandle(Session session, String tableName) { Metadata metadata = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getMetadata(); return transaction(getQueryRunner().getTransactionManager(), getQueryRunner().getAccessControl()) .execute(session, transactionSession -> { QualifiedObjectName objectName = new QualifiedObjectName(catalog, TPCH_SCHEMA, tableName); Optional<TableHandle> handle = metadata.getTableHandle(transactionSession, objectName); InsertTableHandle insertTableHandle = metadata.beginInsert(transactionSession, handle.get()); HiveInsertTableHandle hiveInsertTableHandle = (HiveInsertTableHandle) insertTableHandle.getConnectorHandle(); metadata.finishInsert(transactionSession, insertTableHandle, ImmutableList.of(), ImmutableList.of()); return hiveInsertTableHandle; }); }
public static void main(String[] args) throws Exception { Logging.initialize(); DistributedQueryRunner queryRunner = createQueryRunner(ImmutableMap.of("http-server.http.port", "8080")); Thread.sleep(10); Logger log = Logger.get(BlackHoleQueryRunner.class); log.info("======== SERVER STARTED ========"); log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); } }