public Builder setDataSource(Query query) { this.dataSource = new QueryDataSource(query); return this; }
private Query findInnerMostQuery(Query outerQuery) { Query query = outerQuery; while (query.getDataSource() instanceof QueryDataSource) { query = ((QueryDataSource) query.getDataSource()).getQuery(); } return query; }
@Nullable @Override public DruidQuery toDruidQuery(final boolean finalizeAggregations) { // Must finalize aggregations on subqueries. final DruidQuery subQuery = ((DruidRel) sourceRel).toDruidQuery(true); if (subQuery == null) { return null; } final RowSignature sourceRowSignature = subQuery.getOutputRowSignature(); return partialQuery.build( new QueryDataSource(subQuery.toGroupByQuery()), sourceRowSignature, getPlannerContext(), getCluster().getRexBuilder(), finalizeAggregations ); }
private DataSource getInnerMostDataSource(DataSource dataSource) { if (dataSource instanceof QueryDataSource) { return getInnerMostDataSource(((QueryDataSource) dataSource).getQuery().getDataSource()); } return dataSource; }
@Test public void testQueryDataSource() throws IOException { GroupByQuery query = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions(new DefaultDimensionSpec("quality", "alias")) .setAggregatorSpecs(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index")) .setGranularity(QueryRunnerTestHelper.dayGran) .build(); String dataSourceJSON = "{\"type\":\"query\", \"query\":" + jsonMapper.writeValueAsString(query) + "}"; DataSource dataSource = jsonMapper.readValue(dataSourceJSON, DataSource.class); Assert.assertEquals(new QueryDataSource(query), dataSource); }
private static int countRequiredMergeBufferNum(Query query, int foundNum) { // Note: A broker requires merge buffers for processing the groupBy layers beyond the inner-most one. // For example, the number of required merge buffers for a nested groupBy (groupBy -> groupBy -> table) is 1. // If the broker processes an outer groupBy which reads input from an inner groupBy, // it requires two merge buffers for inner and outer groupBys to keep the intermediate result of inner groupBy // until the outer groupBy processing completes. // This is same for subsequent groupBy layers, and thus the maximum number of required merge buffers becomes 2. final DataSource dataSource = query.getDataSource(); if (foundNum == MAX_MERGE_BUFFER_NUM + 1 || !(dataSource instanceof QueryDataSource)) { return foundNum - 1; } else { return countRequiredMergeBufferNum(((QueryDataSource) dataSource).getQuery(), foundNum + 1); } }
GroupByQuery.builder() .setDataSource( new QueryDataSource( GroupByQuery.builder() .setDataSource(CalciteTests.DATASOURCE1)
@VisibleForTesting public static QuerySegmentSpec getQuerySegmentSpecForLookUp(BaseQuery query) { if (query.getDataSource() instanceof QueryDataSource) { QueryDataSource ds = (QueryDataSource) query.getDataSource(); Query subquery = ds.getQuery(); if (subquery instanceof BaseQuery) { return getQuerySegmentSpecForLookUp((BaseQuery) subquery); } throw new IllegalStateException("Invalid subquery type " + subquery.getClass()); } return query.getQuerySegmentSpec(); }
.builder() .setDataSource( new QueryDataSource( GroupByQuery.builder() .setDataSource(QueryRunnerTestHelper.dataSource)
private Object findInnerDatasource(Query query) { DataSource _ds = query.getDataSource(); if (_ds instanceof TableDataSource) { return ((TableDataSource) _ds).getName(); } if (_ds instanceof QueryDataSource) { return findInnerDatasource(((QueryDataSource) _ds).getQuery()); } if (_ds instanceof UnionDataSource) { return Joiner.on(",") .join( ((UnionDataSource) _ds) .getDataSources() .stream() .map(TableDataSource::getName) .collect(Collectors.toList()) ); } else { // should not come here return query.getDataSource(); } }
@Test(timeout = 60_000L) public void testNotEnoughMergeBuffersOnQueryable() { expectedException.expect(QueryInterruptedException.class); expectedException.expectCause(CoreMatchers.instanceOf(TimeoutException.class)); expectedException.expectMessage("Cannot acquire enough merge buffers"); final GroupByQuery query = GroupByQuery .builder() .setDataSource( new QueryDataSource( GroupByQuery.builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval(QueryRunnerTestHelper.firstToThird) .setGranularity(Granularities.ALL) .setDimensions(new DefaultDimensionSpec("quality", "alias")) .setAggregatorSpecs(Collections.singletonList(QueryRunnerTestHelper.rowsCount)) .build() ) ) .setGranularity(Granularities.ALL) .setInterval(QueryRunnerTestHelper.firstToThird) .setAggregatorSpecs(new LongSumAggregatorFactory("rows", "rows")) .setContext(ImmutableMap.of(QueryContexts.TIMEOUT_KEY, 500)) .build(); GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); }
default List<Interval> getIntervalsOfInnerMostQuery() { if (getDataSource() instanceof QueryDataSource) { //noinspection unchecked return ((QueryDataSource) getDataSource()).getQuery().getIntervalsOfInnerMostQuery(); } else { return getIntervals(); } } }
@Test public void testNestedGroupBy() { final GroupByQuery query = GroupByQuery .builder() .setDataSource( new QueryDataSource( GroupByQuery.builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval(QueryRunnerTestHelper.firstToThird) .setGranularity(Granularities.ALL) .setDimensions(new DefaultDimensionSpec("quality", "alias")) .setAggregatorSpecs(Collections.singletonList(QueryRunnerTestHelper.rowsCount)) .build() ) ) .setGranularity(Granularities.ALL) .setInterval(QueryRunnerTestHelper.firstToThird) .setAggregatorSpecs(new LongSumAggregatorFactory("rows", "rows")) .setContext(ImmutableMap.of(QueryContexts.TIMEOUT_KEY, TIMEOUT)) .build(); GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); assertEquals(1, mergeBufferPool.getMinRemainBufferNum()); assertEquals(3, mergeBufferPool.getPoolSize()); }
GroupByQuery.builder() .setDataSource( new QueryDataSource( GroupByQuery.builder() .setDataSource(CalciteTests.DATASOURCE1)
@Override public Sequence<T> run(final QueryPlus<T> queryPlus, Map<String, Object> responseContext) { DataSource dataSource = queryPlus.getQuery().getDataSource(); boolean forcePushDownNestedQuery = queryPlus.getQuery() .getContextBoolean( GroupByQueryConfig.CTX_KEY_FORCE_PUSH_DOWN_NESTED_QUERY, false ); if (dataSource instanceof QueryDataSource && !forcePushDownNestedQuery) { return run(queryPlus.withQuery((Query<T>) ((QueryDataSource) dataSource).getQuery()), responseContext); } else { QueryPlus newQuery = queryPlus; if (forcePushDownNestedQuery) { // Disable any more push downs before firing off the query. But do let the historical know // that it is executing the complete nested query and not just the inner most part of it newQuery = queryPlus.withQuery( queryPlus.getQuery() .withOverriddenContext( ImmutableMap.of( GroupByQueryConfig.CTX_KEY_FORCE_PUSH_DOWN_NESTED_QUERY, false, GroupByQueryConfig.CTX_KEY_EXECUTING_NESTED_QUERY, true ) ) ); } return baseRunner.run(newQuery, responseContext); } } }
GroupByQuery.builder() .setDataSource( new QueryDataSource( GroupByQuery.builder() .setDataSource(CalciteTests.DATASOURCE1)
private Query findInnerMostQuery(Query outerQuery) { Query query = outerQuery; while (query.getDataSource() instanceof QueryDataSource) { query = ((QueryDataSource) query.getDataSource()).getQuery(); } return query; }
GroupByQuery.builder() .setDataSource( new QueryDataSource( GroupByQuery.builder() .setDataSource(CalciteTests.DATASOURCE1)
@VisibleForTesting public static QuerySegmentSpec getQuerySegmentSpecForLookUp(BaseQuery query) { if (query.getDataSource() instanceof QueryDataSource) { QueryDataSource ds = (QueryDataSource) query.getDataSource(); Query subquery = ds.getQuery(); if (subquery instanceof BaseQuery) { return getQuerySegmentSpecForLookUp((BaseQuery) subquery); } throw new IllegalStateException("Invalid subquery type " + subquery.getClass()); } return query.getQuerySegmentSpec(); }