final List<EventHolder> arg1Val = arg1.getValue().getEvents(); final List<EventHolder> arg2Val = arg2.getValue().getEvents(); builder.addDimensions(mergeColumns(arg1.getValue().getDimensions(), arg2.getValue().getDimensions())); builder.addMetrics(mergeColumns(arg1.getValue().getMetrics(), arg2.getValue().getMetrics()));
public Result<SelectResultValue> build() { return new Result<>(timestamp, new SelectResultValue(pagingIdentifiers, dimensions, metrics, getEventHolders())); }
/** * Don't run this benchmark with a query that doesn't use {@link Granularities#ALL}, * this pagination function probably doesn't work correctly in that case. */ private SelectQuery incrementQueryPagination(SelectQuery query, SelectResultValue prevResult) { Map<String, Integer> pagingIdentifiers = prevResult.getPagingIdentifiers(); Map<String, Integer> newPagingIdentifers = new HashMap<>(); for (String segmentId : pagingIdentifiers.keySet()) { int newOffset = pagingIdentifiers.get(segmentId) + 1; newPagingIdentifers.put(segmentId, newOffset); } return query.withPagingSpec(new PagingSpec(newPagingIdentifers, pagingThreshold)); }
for (Map.Entry<String, Integer> entry : expected.getValue().getPagingIdentifiers().entrySet()) { Assert.assertEquals(entry.getValue(), actual.getValue().getPagingIdentifiers().get(entry.getKey())); Assert.assertEquals(expected.getValue().getDimensions(), actual.getValue().getDimensions()); Assert.assertEquals(expected.getValue().getMetrics(), actual.getValue().getMetrics()); Iterator<EventHolder> expectedEvts = expected.getValue().getEvents().iterator(); Iterator<EventHolder> actualEvts = actual.getValue().getEvents().iterator();
new SelectResultValue( ImmutableMap.of(), Sets.newHashSet("first", "fourth"), new SelectResultValue( ImmutableMap.of(), Sets.newHashSet("second", "third"), Iterator<String> acSegmentIter = merged.getValue().getPagingIdentifiers().keySet().iterator(); Iterator<Integer> acOffsetIter = merged.getValue().getPagingIdentifiers().values().iterator(); List<EventHolder> acEvents = merged.getValue().getEvents();
new SelectResultValue( ImmutableMap.of(), Sets.newHashSet("first", "second", "fourth"), new SelectResultValue( ImmutableMap.of(), Sets.newHashSet("third", "second", "fifth"), Set<String> exMetrics = Sets.newHashSet("eight", "nineth", "seventh"); Set<String> acDimensions = merged.getValue().getDimensions(); Set<String> acMetrics = merged.getValue().getMetrics();
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void queryIncrementalIndex(Blackhole blackhole) { SelectQuery queryCopy = query.withPagingSpec(PagingSpec.newSpec(pagingThreshold)); SegmentId segmentId = SegmentId.dummy("incIndex"); QueryRunner<Row> runner = QueryBenchmarkUtil.makeQueryRunner( factory, segmentId, new IncrementalIndexSegment(incIndexes.get(0), segmentId) ); boolean done = false; while (!done) { List<Result<SelectResultValue>> results = SelectBenchmark.runQuery(factory, runner, queryCopy); SelectResultValue result = results.get(0).getValue(); if (result.getEvents().size() == 0) { done = true; } else { for (EventHolder eh : result.getEvents()) { blackhole.consume(eh); } queryCopy = incrementQueryPagination(queryCopy, result); } } }
private void runAllGranularityTest(SelectQuery query, int[][] expectedOffsets) { for (int[] expected : expectedOffsets) { List<Result<SelectResultValue>> results = runner.run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); Assert.assertEquals(1, results.size()); SelectResultValue value = results.get(0).getValue(); Map<String, Integer> pagingIdentifiers = value.getPagingIdentifiers(); Map<String, Integer> merged = PagingSpec.merge(Collections.singletonList(pagingIdentifiers)); for (int i = 0; i < 4; i++) { if (query.isDescending() ^ expected[i] >= 0) { Assert.assertEquals(expected[i], pagingIdentifiers.get(segmentIdentifiers.get(i)).intValue()); } } Assert.assertEquals(expected[4], value.getEvents().size()); query = query.withPagingSpec(toNextCursor(merged, query, 3)); } }
private static Iterable<Result<SelectResultValue>> populateNullColumnAtLastForQueryableIndexCase(Iterable<Result<SelectResultValue>> results, String columnName) { // A Queryable index does not have the null column when it has loaded a index. for (Result<SelectResultValue> value : results) { Set<String> dimensions = value.getValue().getDimensions(); if (dimensions.contains(columnName)) { break; } dimensions.add(columnName); } return results; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void queryQueryableIndex(Blackhole blackhole) { SelectQuery queryCopy = query.withPagingSpec(PagingSpec.newSpec(pagingThreshold)); SegmentId segmentId = SegmentId.dummy("qIndex"); QueryRunner<Result<SelectResultValue>> runner = QueryBenchmarkUtil.makeQueryRunner( factory, segmentId, new QueryableIndexSegment(qIndexes.get(0), segmentId) ); boolean done = false; while (!done) { List<Result<SelectResultValue>> results = SelectBenchmark.runQuery(factory, runner, queryCopy); SelectResultValue result = results.get(0).getValue(); if (result.getEvents().size() == 0) { done = true; } else { for (EventHolder eh : result.getEvents()) { blackhole.consume(eh); } queryCopy = incrementQueryPagination(queryCopy, result); } } }
final List<EventHolder> arg1Val = arg1.getValue().getEvents(); final List<EventHolder> arg2Val = arg2.getValue().getEvents(); builder.addDimensions(mergeColumns(arg1.getValue().getDimensions(), arg2.getValue().getDimensions())); builder.addMetrics(mergeColumns(arg1.getValue().getMetrics(), arg2.getValue().getMetrics()));
private Iterable<Result<SelectResultValue>> makeSelectResults( Set<String> dimensions, Set<String> metrics, Object... objects ) { List<Result<SelectResultValue>> retVal = new ArrayList<>(); int index = 0; while (index < objects.length) { DateTime timestamp = (DateTime) objects[index++]; List<EventHolder> values = new ArrayList<>(); while (index < objects.length && !(objects[index] instanceof DateTime)) { values.add(new EventHolder(null, 0, (Map) objects[index++])); } Result<SelectResultValue> result = new Result<>( timestamp, new SelectResultValue(ImmutableMap.of(timestamp.toString(), 0), dimensions, metrics, values) ); retVal.add(result); } return retVal; }
private void runDayGranularityTest(SelectQuery query, int[][] expectedOffsets) { for (int[] expected : expectedOffsets) { List<Result<SelectResultValue>> results = runner.run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); Assert.assertEquals(2, results.size()); SelectResultValue value0 = results.get(0).getValue(); SelectResultValue value1 = results.get(1).getValue(); Map<String, Integer> pagingIdentifiers0 = value0.getPagingIdentifiers(); Map<String, Integer> pagingIdentifiers1 = value1.getPagingIdentifiers(); Map<String, Integer> merged = PagingSpec.merge(Arrays.asList(pagingIdentifiers0, pagingIdentifiers1)); for (int i = 0; i < 4; i++) { if (query.isDescending() ^ expected[i] >= 0) { Assert.assertEquals(expected[i], merged.get(segmentIdentifiers.get(i)).intValue()); } } query = query.withPagingSpec(toNextCursor(merged, query, 3)); } }
@Test public void testSimpleDataIngestAndSelectQuery() throws Exception { SketchModule sm = new SketchModule(); sm.configure(null); AggregationTestHelper selectQueryAggregationTestHelper = AggregationTestHelper.createSelectQueryAggregationTestHelper( sm.getJacksonModules(), tempFolder ); Sequence seq = selectQueryAggregationTestHelper.runQueryOnSegments( ImmutableList.of(s1, s2), readFileFromClasspathAsString("select_query.json") ); Result<SelectResultValue> result = (Result<SelectResultValue>) Iterables.getOnlyElement(seq.toList()); Assert.assertEquals(DateTimes.of("2014-10-20T00:00:00.000Z"), result.getTimestamp()); Assert.assertEquals(100, result.getValue().getEvents().size()); Assert.assertEquals("AgMDAAAazJMCAAAAAACAPzz9j7pWTMdROWGf15uY1nI=", result.getValue().getEvents().get(0).getEvent().get("pty_country")); }
new Result( new DateTime(group.get(0).get(EventHolder.timestampKey), ISOChronology.getInstanceUTC()), new SelectResultValue( ImmutableMap.of(segmentId, lastOffset), Sets.newHashSet(dimensions),
@Test public void testSequentialPaging() { int[] asc = {2, 5, 8, 11, 14, 17, 20, 23, 25}; int[] dsc = {-3, -6, -9, -12, -15, -18, -21, -24, -26}; int[] expected = descending ? dsc : asc; SelectQuery query = newTestQuery().intervals(I_0112_0114_SPEC).build(); for (int offset : expected) { List<Result<SelectResultValue>> results = runner.run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); Assert.assertEquals(1, results.size()); SelectResultValue result = results.get(0).getValue(); Map<String, Integer> pagingIdentifiers = result.getPagingIdentifiers(); Assert.assertEquals(offset, pagingIdentifiers.get(SEGMENT_ID_I_0112_0114.toString()).intValue()); Map<String, Integer> next = PagingSpec.next(pagingIdentifiers, descending); query = query.withPagingSpec(new PagingSpec(next, 3, false)); } query = newTestQuery().intervals(I_0112_0114_SPEC).build(); for (int offset : expected) { List<Result<SelectResultValue>> results = runner.run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); Assert.assertEquals(1, results.size()); SelectResultValue result = results.get(0).getValue(); Map<String, Integer> pagingIdentifiers = result.getPagingIdentifiers(); Assert.assertEquals(offset, pagingIdentifiers.get(SEGMENT_ID_I_0112_0114.toString()).intValue()); // use identifier as-is but with fromNext=true query = query.withPagingSpec(new PagingSpec(pagingIdentifiers, 3, true)); } }
new Result<SelectResultValue>( DateTimes.of("2011-01-12T00:00:00.000Z"), new SelectResultValue( ImmutableMap.of(segmentIdString, 2), Sets.newHashSet("null_column", "floatIndex", "longTime"), new Result<SelectResultValue>( DateTimes.of("2011-01-12T00:00:00.000Z"), new SelectResultValue( ImmutableMap.of(segmentIdString, -3), Sets.newHashSet("null_column", "floatIndex", "longTime"),
@Test public void testPagingIdentifiersForUnionDatasource() { Druids.SelectQueryBuilder selectQueryBuilder = Druids .newSelectQueryBuilder() .dataSource( new UnionDataSource( ImmutableList.of( new TableDataSource(QueryRunnerTestHelper.dataSource), new TableDataSource("testing-2") ) ) ) .intervals(SelectQueryRunnerTest.I_0112_0114_SPEC) .granularity(QueryRunnerTestHelper.allGran) .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.dimensions)) .pagingSpec(PagingSpec.newSpec(3)); SelectQuery query = selectQueryBuilder.build(); QueryRunner unionQueryRunner = new UnionQueryRunner(runner); List<Result<SelectResultValue>> results = unionQueryRunner.run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); Map<String, Integer> pagingIdentifiers = results.get(0).getValue().getPagingIdentifiers(); query = query.withPagingSpec(toNextCursor(PagingSpec.merge(Collections.singletonList(pagingIdentifiers)), query, 3)); unionQueryRunner.run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); }
new Result<SelectResultValue>( DateTimes.of("2011-01-12T00:00:00.000Z"), new SelectResultValue( ImmutableMap.of(segmentIdString, 2), Sets.newHashSet("null_column", "floatIndex", "longTime"), new Result<SelectResultValue>( DateTimes.of("2011-01-12T00:00:00.000Z"), new SelectResultValue( ImmutableMap.of(segmentIdString, -3), Sets.newHashSet("null_column", "floatIndex", "longTime"),