private static <T> Sequence<T> executeQuery(final Injector injector, final QueryableIndex index, final Query<T> query) { final QueryRunnerFactoryConglomerate conglomerate = injector.getInstance(QueryRunnerFactoryConglomerate.class); final QueryRunnerFactory<T, Query<T>> factory = conglomerate.findFactory(query); final QueryRunner<T> runner = factory.createRunner(new QueryableIndexSegment(index, SegmentId.dummy("segment"))); return factory .getToolchest() .mergeResults(factory.mergeRunners(Execs.directExecutor(), ImmutableList.of(runner))) .run(QueryPlus.wrap(query), new HashMap<>()); }
@Override public <T> QueryRunner<T> getQueryRunnerForIntervals(final Query<T> query, Iterable<Interval> intervals) { final QueryRunnerFactory<T, Query<T>> factory = conglomerate.findFactory(query); final Map<Integer, FireChief> partitionChiefs = chiefs.get(Iterables.getOnlyElement(query.getDataSource() .getNames())); return partitionChiefs == null ? new NoopQueryRunner<T>() : factory.getToolchest().mergeResults( factory.mergeRunners( Execs.directExecutor(), // Chaining query runners which wait on submitted chain query runners can make executor pools deadlock Iterables.transform( partitionChiefs.values(), new Function<FireChief, QueryRunner<T>>() { @Override public QueryRunner<T> apply(FireChief fireChief) { return fireChief.getQueryRunner(query); } } ) ) ); }
@Override public <T> QueryRunner<T> getQueryRunnerForSegments(final Query<T> query, final Iterable<SegmentDescriptor> specs) { final QueryRunnerFactory<T, Query<T>> factory = conglomerate.findFactory(query); final Map<Integer, FireChief> partitionChiefs = chiefs.get(Iterables.getOnlyElement(query.getDataSource() .getNames())); return partitionChiefs == null ? new NoopQueryRunner<T>() : factory.getToolchest().mergeResults( factory.mergeRunners( Execs.directExecutor(), Iterables.transform( specs, new Function<SegmentDescriptor, QueryRunner<T>>() { @Override public QueryRunner<T> apply(SegmentDescriptor spec) { final FireChief retVal = partitionChiefs.get(spec.getPartitionNumber()); return retVal == null ? new NoopQueryRunner<T>() : retVal.getQueryRunner(query.withQuerySegmentSpec(new SpecificSegmentSpec(spec))); } } ) ) ); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void queryMultiQueryableIndexWithSpilling(Blackhole blackhole) { QueryToolChest<Row, GroupByQuery> toolChest = factory.getToolchest(); QueryRunner<Row> theRunner = new FinalizeResultsQueryRunner<>( toolChest.mergeResults( factory.mergeRunners(executorService, makeMultiRunners()) ), (QueryToolChest) toolChest ); final GroupByQuery spillingQuery = query.withOverriddenContext( ImmutableMap.of("bufferGrouperMaxSize", 4000) ); Sequence<Row> queryResult = theRunner.run(QueryPlus.wrap(spillingQuery), new HashMap<>()); List<Row> results = queryResult.toList(); for (Row result : results) { blackhole.consume(result); } }
toolChest.mergeResults(factory.mergeRunners(executorService, singleSegmentRunners)), toolChest
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void queryMultiQueryableIndex(Blackhole blackhole) { QueryToolChest<Row, GroupByQuery> toolChest = factory.getToolchest(); QueryRunner<Row> theRunner = new FinalizeResultsQueryRunner<>( toolChest.mergeResults( factory.mergeRunners(executorService, makeMultiRunners()) ), (QueryToolChest) toolChest ); Sequence<Row> queryResult = theRunner.run(QueryPlus.wrap(query), new HashMap<>()); List<Row> results = queryResult.toList(); for (Row result : results) { blackhole.consume(result); } }
toolChest.mergeResults(factory.mergeRunners(executorService, singleSegmentRunners)), toolChest
toolChest.mergeResults(factory.mergeRunners(executorService, singleSegmentRunners)), toolChest
toolChest.mergeResults(factory.mergeRunners(exec, queryRunners)), toolChest ),
toolChest.mergeResults(factory.mergeRunners(executorService, singleSegmentRunners)), toolChest
@Test public void testMergeRunnersWithLimit() { ScanQuery query = newBuilder().build(); List<ScanResultValue> results = factory .mergeRunners( Execs.directExecutor(), ImmutableList.of(factory.createRunner(segment0), factory.createRunner(segment1)) ) .run(QueryPlus.wrap(query), new HashMap<>()) .toList(); int totalCount = 0; for (ScanResultValue result : results) { System.out.println(((List) result.getEvents()).size()); totalCount += ((List) result.getEvents()).size(); } Assert.assertEquals( totalCount, limit != 0 ? Math.min(limit, V_0112.length + V_0113.length) : V_0112.length + V_0113.length ); }
toolChest.mergeResults( toolChest.preMergeQueryDecoration( factory.mergeRunners( Execs.directExecutor(), Lists.transform(
toolChest.mergeResults(factory.mergeRunners(exec, queryRunners)), toolChest ),
return factory.getToolchest() .mergeResults( factory.mergeRunners( Execs.directExecutor(), Iterables.transform(
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void queryMultiQueryableIndexWithSerde(Blackhole blackhole) { QueryToolChest<Row, GroupByQuery> toolChest = factory.getToolchest(); QueryRunner<Row> theRunner = new FinalizeResultsQueryRunner<>( toolChest.mergeResults( new SerializingQueryRunner<>( new DefaultObjectMapper(new SmileFactory()), Row.class, toolChest.mergeResults( factory.mergeRunners(executorService, makeMultiRunners()) ) ) ), (QueryToolChest) toolChest ); Sequence<Row> queryResult = theRunner.run(QueryPlus.wrap(query), new HashMap<>()); List<Row> results = queryResult.toList(); for (Row result : results) { blackhole.consume(result); } }
public static <T, QueryType extends Query<T>> List<T> runQuery( final QueryType query, final QueryRunnerFactory<T, QueryType> factory, final List<QueryableIndex> indexes ) { final Sequence<T> results = new FinalizeResultsQueryRunner<>( factory.getToolchest().mergeResults( factory.mergeRunners( Execs.directExecutor(), FunctionalIterable .create(indexes) .transform( index -> factory.createRunner(new QueryableIndexSegment(index, SegmentId.dummy("xxx"))) ) ) ), (QueryToolChest<T, Query<T>>) factory.getToolchest() ).run(QueryPlus.wrap(query), new HashMap<>()); return results.toList(); }
factory.mergeRunners( Execs.directExecutor(), FunctionalIterable
QueryRunner<Row> theRunner = new FinalizeResultsQueryRunner<>( toolChest.mergeResults( groupByFactory.mergeRunners(executorService, getRunner1()) ), (QueryToolChest) toolChest tooSmallGroupByFactory.mergeRunners(executorService, getRunner2()) ), (QueryToolChest) toolChest
QueryRunner<Row> theRunner = new FinalizeResultsQueryRunner<>( toolChest.mergeResults( groupByFactory.mergeRunners(executorService, makeGroupByMultiRunners()) ), (QueryToolChest) toolChest
QueryRunner<Row> segment1Runner = new FinalizeResultsQueryRunner<Row>( toolChest.mergeResults( groupByFactory.mergeRunners(executorService, getQueryRunnerForSegment1()) ), (QueryToolChest) toolChest groupByFactory2.mergeRunners(executorService, getQueryRunnerForSegment2()) ), (QueryToolChest) toolChest