@Provides private HttpRemoteTaskFactory createHttpRemoteTaskFactory( JsonMapper jsonMapper, JsonCodec<TaskStatus> taskStatusCodec, JsonCodec<TaskInfo> taskInfoCodec, JsonCodec<TaskUpdateRequest> taskUpdateRequestCodec) { JaxrsTestingHttpProcessor jaxrsTestingHttpProcessor = new JaxrsTestingHttpProcessor(URI.create("http://fake.invalid/"), testingTaskResource, jsonMapper); TestingHttpClient testingHttpClient = new TestingHttpClient(jaxrsTestingHttpProcessor.setTrace(TRACE_HTTP)); testingTaskResource.setHttpClient(testingHttpClient); return new HttpRemoteTaskFactory( new QueryManagerConfig(), TASK_MANAGER_CONFIG, testingHttpClient, new TestSqlTaskManager.MockLocationFactory(), taskStatusCodec, taskInfoCodec, taskUpdateRequestCodec, new RemoteTaskStats()); } });
private QueryExplainer getQueryExplainer() { Metadata metadata = queryRunner.getMetadata(); FeaturesConfig featuresConfig = new FeaturesConfig().setOptimizeHashGeneration(true); boolean forceSingleNode = queryRunner.getNodeCount() == 1; TaskCountEstimator taskCountEstimator = new TaskCountEstimator(queryRunner::getNodeCount); CostCalculator costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator); List<PlanOptimizer> optimizers = new PlanOptimizers( metadata, sqlParser, featuresConfig, forceSingleNode, new MBeanExporter(new TestingMBeanServer()), queryRunner.getSplitManager(), queryRunner.getPageSourceManager(), queryRunner.getStatsCalculator(), costCalculator, new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator), new CostComparator(featuresConfig), taskCountEstimator).get(); return new QueryExplainer( optimizers, new PlanFragmenter(metadata, queryRunner.getNodePartitioningManager(), new QueryManagerConfig()), metadata, queryRunner.getAccessControl(), sqlParser, queryRunner.getStatsCalculator(), costCalculator, ImmutableMap.of()); }
.build(); QueryManagerConfig expected = new QueryManagerConfig() .setMinQueryExpireAge(new Duration(30, TimeUnit.SECONDS)) .setMaxQueryHistory(10)
@Test public void testTooManyGroupingElements() { Session session = testSessionBuilder(new SessionPropertyManager(new SystemSessionProperties( new QueryManagerConfig(), new TaskManagerConfig(), new MemoryManagerConfig(), new FeaturesConfig().setMaxGroupingSets(2048)))).build(); analyze(session, "SELECT a, b, c, d, e, f, g, h, i, j, k, SUM(l)" + "FROM (VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12))\n" + "t (a, b, c, d, e, f, g, h, i, j, k, l)\n" + "GROUP BY CUBE (a, b, c, d, e, f), CUBE (g, h, i, j, k)"); assertFails(session, TOO_MANY_GROUPING_SETS, "line 3:10: GROUP BY has 4096 grouping sets but can contain at most 2048", "SELECT a, b, c, d, e, f, g, h, i, j, k, l, SUM(m)" + "FROM (VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13))\n" + "t (a, b, c, d, e, f, g, h, i, j, k, l, m)\n" + "GROUP BY CUBE (a, b, c, d, e, f), CUBE (g, h, i, j, k, l)"); assertFails(session, TOO_MANY_GROUPING_SETS, format("line 3:10: GROUP BY has more than %s grouping sets but can contain at most 2048", Integer.MAX_VALUE), "SELECT a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, " + "q, r, s, t, u, v, x, w, y, z, aa, ab, ac, ad, ae, SUM(af)" + "FROM (VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, " + "17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32))\n" + "t (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, " + "q, r, s, t, u, v, x, w, y, z, aa, ab, ac, ad, ae, af)\n" + "GROUP BY CUBE (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, " + "q, r, s, t, u, v, x, w, y, z, aa, ab, ac, ad, ae)"); }
this.nodePartitioningManager = new NodePartitioningManager(nodeScheduler); this.splitManager = new SplitManager(new QueryManagerConfig()); this.blockEncodingManager = new BlockEncodingManager(typeRegistry); this.metadata = new MetadataManager( typeRegistry, blockEncodingManager, new SessionPropertyManager(new SystemSessionProperties(new QueryManagerConfig(), new TaskManagerConfig(), new MemoryManagerConfig(), featuresConfig)), new SchemaPropertyManager(), new TablePropertyManager(), new ColumnPropertyManager(), transactionManager); this.planFragmenter = new PlanFragmenter(this.metadata, this.nodePartitioningManager, new QueryManagerConfig()); this.joinCompiler = new JoinCompiler(metadata, featuresConfig); this.pageIndexerFactory = new GroupByHashPageIndexerFactory(joinCompiler);
@BeforeClass public void setUp() { TaskCountEstimator taskCountEstimator = new TaskCountEstimator(() -> NUMBER_OF_NODES); costCalculatorUsingExchanges = new CostCalculatorUsingExchanges(taskCountEstimator); costCalculatorWithEstimatedExchanges = new CostCalculatorWithEstimatedExchanges(costCalculatorUsingExchanges, taskCountEstimator); session = testSessionBuilder().setCatalog("tpch").build(); CatalogManager catalogManager = new CatalogManager(); catalogManager.registerCatalog(createBogusTestingCatalog("tpch")); transactionManager = createTestTransactionManager(catalogManager); metadata = createTestMetadataManager(transactionManager, new FeaturesConfig()); finalizerService = new FinalizerService(); finalizerService.start(); nodeScheduler = new NodeScheduler( new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)); nodePartitioningManager = new NodePartitioningManager(nodeScheduler); planFragmenter = new PlanFragmenter(metadata, nodePartitioningManager, new QueryManagerConfig()); }
@Inject public SystemSessionProperties( QueryManagerConfig queryManagerConfig, TaskManagerConfig taskManagerConfig, MemoryManagerConfig memoryManagerConfig, FeaturesConfig featuresConfig) { sessionProperties = ImmutableList.of( stringProperty( EXECUTION_POLICY, "Policy used for scheduling query tasks", queryManagerConfig.getQueryExecutionPolicy(), false), booleanProperty( OPTIMIZE_HASH_GENERATION, "Compute hash codes for distribution, joins, and aggregations early in query plan", featuresConfig.isOptimizeHashGeneration(), false), booleanProperty( DISTRIBUTED_JOIN, "(DEPRECATED) Use a distributed join instead of a broadcast join. If this is set, join_distribution_type is ignored.", null, false), new PropertyMetadata<>( JOIN_DISTRIBUTION_TYPE, format("The join method to use. Options are %s", Stream.of(JoinDistributionType.values()) .map(JoinDistributionType::name) .collect(joining(","))), VARCHAR,
private void parse(String fileName) { String path = this.getClass().getClassLoader().getResource(fileName).getPath(); QueryManagerConfig config = new QueryManagerConfig(); config.setQueueConfigFile(path); new SqlQueryQueueManager(config, new ObjectMapperProvider().get(), new MBeanExporter(ManagementFactory.getPlatformMBeanServer())); }
.build(); QueryManagerConfig expected = new QueryManagerConfig() .setMaxQueryAge(new Duration(30, TimeUnit.SECONDS)) .setMaxQueryHistory(10)
private QueryExplainer getQueryExplainer() { Metadata metadata = queryRunner.getMetadata(); FeaturesConfig featuresConfig = new FeaturesConfig().setOptimizeHashGeneration(true); boolean forceSingleNode = queryRunner.getNodeCount() == 1; TaskCountEstimator taskCountEstimator = new TaskCountEstimator(queryRunner::getNodeCount); CostCalculator costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator); List<PlanOptimizer> optimizers = new PlanOptimizers( metadata, sqlParser, featuresConfig, forceSingleNode, new MBeanExporter(new TestingMBeanServer()), queryRunner.getSplitManager(), queryRunner.getPageSourceManager(), queryRunner.getStatsCalculator(), costCalculator, new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator), new CostComparator(featuresConfig), taskCountEstimator).get(); return new QueryExplainer( optimizers, new PlanFragmenter(metadata, queryRunner.getNodePartitioningManager(), new QueryManagerConfig()), metadata, queryRunner.getAccessControl(), sqlParser, queryRunner.getStatsCalculator(), costCalculator, ImmutableMap.of()); }
@Inject public SystemSessionProperties( QueryManagerConfig queryManagerConfig, TaskManagerConfig taskManagerConfig, MemoryManagerConfig memoryManagerConfig, FeaturesConfig featuresConfig) { sessionProperties = ImmutableList.of( stringSessionProperty( EXECUTION_POLICY, "Policy used for scheduling query tasks", queryManagerConfig.getQueryExecutionPolicy(), false), booleanSessionProperty( OPTIMIZE_HASH_GENERATION, "Compute hash codes for distribution, joins, and aggregations early in query plan", featuresConfig.isOptimizeHashGeneration(), false), booleanSessionProperty( DISTRIBUTED_JOIN, "Use a distributed join instead of a broadcast join", featuresConfig.isDistributedJoinsEnabled(), false), booleanSessionProperty( DISTRIBUTED_INDEX_JOIN, "Distribute index joins on join keys instead of executing inline", featuresConfig.isDistributedIndexJoinsEnabled(), false), integerSessionProperty( HASH_PARTITION_COUNT,