@Override public StorageAdapter asStorageAdapter() { return new QueryableIndexStorageAdapter(index); }
@Override public StorageAdapter toStorageAdapter() { return new QueryableIndexStorageAdapter(this); }
@Override public WindowedStorageAdapter apply(WindowedDataSegment segment) { try { logger.info("Getting storage path for segment [%s]", segment.getSegment().getId()); Path path = new Path(JobHelper.getURIFromSegment(segment.getSegment())); logger.info("Fetch segment files from [%s]", path); File dir = Files.createTempDir(); tmpSegmentDirs.add(dir); logger.info("Locally storing fetched segment at [%s]", dir); JobHelper.unzipNoGuava(path, context.getConfiguration(), dir, context, null); logger.info("finished fetching segment files"); QueryableIndex index = HadoopDruidIndexerConfig.INDEX_IO.loadIndex(dir); indexes.add(index); numRows += index.getNumRows(); return new WindowedStorageAdapter( new QueryableIndexStorageAdapter(index), segment.getInterval() ); } catch (IOException ex) { throw Throwables.propagate(ex); } } }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void longRead(Blackhole blackhole) { StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, null); readCursorsLong(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void timeFilterHalf(Blackhole blackhole) { StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, timeFilterHalf); readCursorsLong(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void stringRead(Blackhole blackhole) { StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, null); readCursors(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void timeFilterNone(Blackhole blackhole) { StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, timeFilterNone); readCursorsLong(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void timeFilterAll(Blackhole blackhole) { StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, timeFilterAll); readCursorsLong(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void readWithPostFilter(Blackhole blackhole) { Filter filter = new NoBitmapSelectorFilter("dimSequential", "199"); StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, filter); readCursors(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void readWithPreFilter(Blackhole blackhole) { Filter filter = new SelectorFilter("dimSequential", "199"); StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, filter); readCursors(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void readOrFilter(Blackhole blackhole) { Filter filter = new NoBitmapSelectorFilter("dimSequential", "199"); Filter filter2 = new AndFilter(Arrays.asList(new SelectorFilter("dimMultivalEnumerated2", "Corundum"), new NoBitmapSelectorFilter("dimMultivalEnumerated", "Bar"))); Filter orFilter = new OrFilter(Arrays.asList(filter, filter2)); StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, orFilter); readCursors(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void readWithExFnPostFilter(Blackhole blackhole) { Filter filter = new NoBitmapSelectorDimFilter("dimSequential", "super-199", JS_EXTRACTION_FN).toFilter(); StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, filter); readCursors(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void readOrFilterCNF(Blackhole blackhole) { Filter filter = new NoBitmapSelectorFilter("dimSequential", "199"); Filter filter2 = new AndFilter(Arrays.asList(new SelectorFilter("dimMultivalEnumerated2", "Corundum"), new NoBitmapSelectorFilter("dimMultivalEnumerated", "Bar"))); Filter orFilter = new OrFilter(Arrays.asList(filter, filter2)); StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, Filters.convertToCNF(orFilter)); readCursors(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void readWithExFnPreFilter(Blackhole blackhole) { Filter filter = new SelectorDimFilter("dimSequential", "super-199", JS_EXTRACTION_FN).toFilter(); StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, filter); readCursors(cursors, blackhole); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void readAndFilter(Blackhole blackhole) { Filter andFilter = new AndFilter( ImmutableList.of( new SelectorFilter("dimUniform", "199"), new NoBitmapSelectorDimFilter("dimUniform", "super-199", JS_EXTRACTION_FN).toFilter() ) ); StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, andFilter); readCursors(cursors, blackhole); }
private double compute(final Function<ColumnSelectorFactory, BufferAggregator> aggregatorFactory) { final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index); final Sequence<Cursor> cursors = adapter.makeCursors( null, index.getDataInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null ); final List<Double> results = cursors .map(cursor -> { final BufferAggregator bufferAggregator = aggregatorFactory.apply(cursor.getColumnSelectorFactory()); bufferAggregator.init(aggregationBuffer, 0); while (!cursor.isDone()) { bufferAggregator.aggregate(aggregationBuffer, 0); cursor.advance(); } final Double dbl = (Double) bufferAggregator.get(aggregationBuffer, 0); bufferAggregator.close(); return dbl; }) .toList(); return Iterables.getOnlyElement(results); }
@Benchmark public void timeFloorUsingCursor(Blackhole blackhole) { final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors( null, index.getDataInterval(), VirtualColumns.EMPTY, Granularities.HOUR, false, null ); final List<Long> results = cursors .map(cursor -> { long count = 0L; while (!cursor.isDone()) { count++; cursor.advance(); } return count; }) .toList(); long count = 0L; for (Long result : results) { count += result; } blackhole.consume(count); }
@Benchmark public void strlenUsingExtractionFn(Blackhole blackhole) { final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors( null, index.getDataInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null ); final List<?> results = cursors .map(cursor -> { final DimensionSelector selector = cursor .getColumnSelectorFactory() .makeDimensionSelector(new ExtractionDimensionSpec("x", "v", StrlenExtractionFn.instance())); consumeDimension(cursor, selector, blackhole); return null; }) .toList(); blackhole.consume(results); }
@Benchmark public void timeFloorUsingExtractionFn(Blackhole blackhole) { final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors( null, index.getDataInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null ); final List<?> results = cursors .map(cursor -> { final DimensionSelector selector = cursor .getColumnSelectorFactory() .makeDimensionSelector( new ExtractionDimensionSpec( ColumnHolder.TIME_COLUMN_NAME, "v", new TimeFormatExtractionFn(null, null, null, Granularities.HOUR, true) ) ); consumeDimension(cursor, selector, blackhole); return null; }) .toList(); blackhole.consume(results); }
@Benchmark public void timeFormatUsingExtractionFn(Blackhole blackhole) { final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors( null, index.getDataInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null ); final List<?> results = cursors .map(cursor -> { final DimensionSelector selector = cursor .getColumnSelectorFactory() .makeDimensionSelector( new ExtractionDimensionSpec( ColumnHolder.TIME_COLUMN_NAME, "v", new TimeFormatExtractionFn("yyyy-MM-dd", null, null, null, false) ) ); consumeDimension(cursor, selector, blackhole); return null; }) .toList(); blackhole.consume(results); }