@Override public StorageAdapter asStorageAdapter() { return new QueryableIndexStorageAdapter(index); }
private void runDump(final Injector injector, final QueryableIndex index) throws IOException final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index); final List<String> columnNames = getColumnsToInclude(index); final DimFilter filter = filterJson != null ? objectMapper.readValue(filterJson, DimFilter.class) : null; final Sequence<Cursor> cursors = adapter.makeCursors( Filters.toFilter(filter), index.getDataInterval().withChronology(ISOChronology.getInstanceUTC()),
@Override @Nullable public ColumnCapabilities getColumnCapabilities(String column) { return getColumnCapabilities(index, column); }
indexSpec ); final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex( merged))); Assert.assertEquals(ImmutableSet.of("A", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
DateTime minTime = getMinTime(); long minDataTimestamp = minTime.getMillis(); DateTime maxTime = getMaxTime(); long maxDataTimestamp = maxTime.getMillis(); final Interval dataInterval = new Interval(minTime, gran.bucketEnd(maxTime));
@Override public DateTime getMaxIngestedEventTime() { // For immutable indexes, maxIngestedEventTime is maxTime. return getMaxTime(); }
indexSpec ); final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex( merged))); Assert.assertEquals(ImmutableSet.of("A", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
DateTime minTime = getMinTime(); long minDataTimestamp = minTime.getMillis(); DateTime maxTime = getMaxTime(); long maxDataTimestamp = maxTime.getMillis(); final Interval dataInterval = new Interval(minTime, gran.bucketEnd(maxTime));
@Override public DateTime getMaxIngestedEventTime() { // For immutable indexes, maxIngestedEventTime is maxTime. return getMaxTime(); }
@Override public StorageAdapter toStorageAdapter() { return new QueryableIndexStorageAdapter(this); }
@Benchmark public void timeFloorUsingCursor(Blackhole blackhole) { final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors( null, index.getDataInterval(), VirtualColumns.EMPTY, Granularities.HOUR, false, null ); final List<Long> results = cursors .map(cursor -> { long count = 0L; while (!cursor.isDone()) { count++; cursor.advance(); } return count; }) .toList(); long count = 0L; for (Long result : results) { count += result; } blackhole.consume(count); }
indexSpec ); final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter( closer.closeLater(indexIO.loadIndex(merged)) ); Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
@Override @Nullable public ColumnCapabilities getColumnCapabilities(String columnName) { if (virtualColumns.exists(columnName)) { return virtualColumns.getColumnCapabilities(columnName); } return QueryableIndexStorageAdapter.getColumnCapabilities(index, columnName); } }
@Override public WindowedStorageAdapter apply(WindowedDataSegment segment) { try { logger.info("Getting storage path for segment [%s]", segment.getSegment().getId()); Path path = new Path(JobHelper.getURIFromSegment(segment.getSegment())); logger.info("Fetch segment files from [%s]", path); File dir = Files.createTempDir(); tmpSegmentDirs.add(dir); logger.info("Locally storing fetched segment at [%s]", dir); JobHelper.unzipNoGuava(path, context.getConfiguration(), dir, context, null); logger.info("finished fetching segment files"); QueryableIndex index = HadoopDruidIndexerConfig.INDEX_IO.loadIndex(dir); indexes.add(index); numRows += index.getNumRows(); return new WindowedStorageAdapter( new QueryableIndexStorageAdapter(index), segment.getInterval() ); } catch (IOException ex) { throw Throwables.propagate(ex); } } }
private double compute(final Function<ColumnSelectorFactory, BufferAggregator> aggregatorFactory) { final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index); final Sequence<Cursor> cursors = adapter.makeCursors( null, index.getDataInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null ); final List<Double> results = cursors .map(cursor -> { final BufferAggregator bufferAggregator = aggregatorFactory.apply(cursor.getColumnSelectorFactory()); bufferAggregator.init(aggregationBuffer, 0); while (!cursor.isDone()) { bufferAggregator.aggregate(aggregationBuffer, 0); cursor.advance(); } final Double dbl = (Double) bufferAggregator.get(aggregationBuffer, 0); bufferAggregator.close(); return dbl; }) .toList(); return Iterables.getOnlyElement(results); }
final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex( merged))); Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
@Override @Nullable public ColumnCapabilities getColumnCapabilities(String column) { return getColumnCapabilities(index, column); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public void longRead(Blackhole blackhole) { StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex); Sequence<Cursor> cursors = makeCursors(sa, null); readCursorsLong(cursors, blackhole); }
@Benchmark public void timeFloorUsingExpression(Blackhole blackhole) { final Sequence<Cursor> cursors = new QueryableIndexStorageAdapter(index).makeCursors( null, index.getDataInterval(), VirtualColumns.create( ImmutableList.of( new ExpressionVirtualColumn( "v", "timestamp_floor(__time, 'PT1H')", ValueType.LONG, TestExprMacroTable.INSTANCE ) ) ), Granularities.ALL, false, null ); final List<?> results = cursors .map(cursor -> { final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v"); consumeLong(cursor, selector, blackhole); return null; }) .toList(); blackhole.consume(results); }
@Override @Nullable public ColumnCapabilities getColumnCapabilities(String columnName) { if (virtualColumns.exists(columnName)) { return virtualColumns.getColumnCapabilities(columnName); } return QueryableIndexStorageAdapter.getColumnCapabilities(index, columnName); } }