public static String createScanAllQuery(String dataSourceName, List<String> columns) throws JsonProcessingException { final ScanQuery.ScanQueryBuilder scanQueryBuilder = ScanQuery.newScanQueryBuilder(); final List<Interval> intervals = Collections.singletonList(DEFAULT_INTERVAL); ScanQuery scanQuery = scanQueryBuilder.dataSource(dataSourceName) .resultFormat(ScanQuery.RESULT_FORMAT_COMPACTED_LIST) .intervals(new MultipleIntervalSegmentSpec(intervals)) .columns(columns) .build(); return JSON_MAPPER.writeValueAsString(scanQuery); }
@Override public boolean nextKeyValue() throws IOException { if (compactedValues.hasNext()) { return true; } if (queryResultsIterator.hasNext()) { ScanResultValue current = queryResultsIterator.next(); //noinspection unchecked compactedValues = ((List<List<Object>>) current.getEvents()).iterator(); return nextKeyValue(); } return false; }
private static HiveDruidSplit[] distributeScanQuery(String address, ScanQuery query, Path dummyPath) throws IOException { final boolean isFetch = query.getLimit() < Long.MAX_VALUE; if (isFetch) { return new HiveDruidSplit[] {new HiveDruidSplit(DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(query), newSD = new SegmentDescriptor(locatedSD.getInterval(), locatedSD.getVersion(), locatedSD.getPartitionNumber()); final Query partialQuery = query.withQuerySegmentSpec(new MultipleSpecificSegmentSpec(Lists.newArrayList(newSD))); splits[i] = new HiveDruidSplit(DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(partialQuery), dummyPath, hosts);
public static ScanQueryBuilder copy(ScanQuery query) { return new ScanQueryBuilder() .dataSource(query.getDataSource()) .intervals(query.getQuerySegmentSpec()) .virtualColumns(query.getVirtualColumns()) .resultFormat(query.getResultFormat()) .batchSize(query.getBatchSize()) .limit(query.getLimit()) .filters(query.getFilter()) .columns(query.getColumns()) .legacy(query.isLegacy()) .context(query.getContext()); }
@Override public Query<ScanResultValue> withOverriddenContext(Map<String, Object> contextOverrides) { return ScanQueryBuilder.copy(this).context(computeOverriddenContext(getContext(), contextOverrides)).build(); }
@Override public Query<ScanResultValue> withDataSource(DataSource dataSource) { return ScanQueryBuilder.copy(this).dataSource(dataSource).build(); }
@Override public ScanResultValue next() { ScanResultValue batch = yielder.get(); if (ScanQuery.RESULT_FORMAT_COMPACTED_LIST.equals(resultFormat) || ScanQuery.RESULT_FORMAT_LIST.equals(resultFormat)) { List events = (List) batch.getEvents(); if (events.size() <= limit - count) { count += events.size(); yielder = yielder.next(null); return batch; } else { // last batch // single batch length is <= Integer.MAX_VALUE, so this should not overflow int left = (int) (limit - count); count = limit; return new ScanResultValue(batch.getSegmentId(), batch.getColumns(), events.subList(0, left)); } } throw new UnsupportedOperationException(ScanQuery.RESULT_FORMAT_VALUE_VECTOR + " is not supported yet"); }
@Override public String toString() { return "ScanQuery{" + "dataSource='" + getDataSource() + '\'' + ", querySegmentSpec=" + getQuerySegmentSpec() + ", virtualColumns=" + getVirtualColumns() + ", resultFormat='" + resultFormat + '\'' + ", batchSize=" + batchSize + ", limit=" + limit + ", dimFilter=" + dimFilter + ", columns=" + columns + ", legacy=" + legacy + '}'; }
public ScanQueryLimitRowIterator( QueryRunner<ScanResultValue> baseRunner, QueryPlus<ScanResultValue> queryPlus, Map<String, Object> responseContext ) { ScanQuery query = (ScanQuery) queryPlus.getQuery(); resultFormat = query.getResultFormat(); limit = query.getLimit(); Sequence<ScanResultValue> baseSequence = baseRunner.run(queryPlus, responseContext); yielder = baseSequence.toYielder( null, new YieldingAccumulator<ScanResultValue, ScanResultValue>() { @Override public ScanResultValue accumulate(ScanResultValue accumulated, ScanResultValue in) { yield(); return in; } } ); }
@Override public Sequence<ScanResultValue> run( final QueryPlus<ScanResultValue> queryPlus, final Map<String, Object> responseContext ) { // Ensure "legacy" is a non-null value, such that all other nodes this query is forwarded to will treat it // the same way, even if they have different default legacy values. final ScanQuery scanQuery = ((ScanQuery) queryPlus.getQuery()).withNonNullLegacy(scanQueryConfig); final QueryPlus<ScanResultValue> queryPlusWithNonNullLegacy = queryPlus.withQuery(scanQuery); if (scanQuery.getLimit() == Long.MAX_VALUE) { return runner.run(queryPlusWithNonNullLegacy, responseContext); } return new BaseSequence<>( new BaseSequence.IteratorMaker<ScanResultValue, ScanQueryLimitRowIterator>() { @Override public ScanQueryLimitRowIterator make() { return new ScanQueryLimitRowIterator(runner, queryPlusWithNonNullLegacy, responseContext); } @Override public void cleanup(ScanQueryLimitRowIterator iterFromMake) { CloseQuietly.close(iterFromMake); } } ); } };
@Override public Sequence<ScanResultValue> run( QueryPlus<ScanResultValue> queryPlus, Map<String, Object> responseContext ) { ScanQuery scanQuery = (ScanQuery) queryPlus.getQuery(); if (scanQuery.getFilter() != null) { scanQuery = scanQuery.withDimFilter(scanQuery.getFilter().optimize()); queryPlus = queryPlus.withQuery(scanQuery); } return runner.run(queryPlus, responseContext); } };
@Override public ScanResultValue accumulate(ScanResultValue accumulated, ScanResultValue in) { yield(); return in; } }
public static ScanQueryBuilder newScanQueryBuilder() { return new ScanQueryBuilder(); } }
@Override public QueryRunner<ScanResultValue> createRunner(Segment segment) { return new ScanQueryRunner(engine, segment); }
public ScanQuery build() { return new ScanQuery( dataSource, querySegmentSpec, virtualColumns, resultFormat, batchSize, limit, dimFilter, columns, legacy, context ); }
public ScanQueryBuilder virtualColumns(VirtualColumn... virtualColumns) { return virtualColumns(VirtualColumns.create(Arrays.asList(virtualColumns))); }
@Override public Sequence<ScanResultValue> run( QueryPlus<ScanResultValue> queryPlus, Map<String, Object> responseContext ) { Query<ScanResultValue> query = queryPlus.getQuery(); if (!(query instanceof ScanQuery)) { throw new ISE("Got a [%s] which isn't a %s", query.getClass(), ScanQuery.class); } // it happens in unit tests final Number timeoutAt = (Number) responseContext.get(CTX_TIMEOUT_AT); if (timeoutAt == null || timeoutAt.longValue() == 0L) { responseContext.put(CTX_TIMEOUT_AT, JodaUtils.MAX_INSTANT); } return engine.process((ScanQuery) query, segment, responseContext); } }