public Query getRealQuery(Query query) { if (query instanceof MaterializedViewQuery) { optimizer = ((MaterializedViewQuery) query).getOptimizer(); return ((MaterializedViewQuery) query).getQuery(); } return query; } }
@Override public int compareTo(DerivativeDataSource o) { if (this.avgSizeBasedGranularity > o.getAvgSizeBasedGranularity()) { return 1; } else if (this.avgSizeBasedGranularity == o.getAvgSizeBasedGranularity()) { return 0; } else { return -1; } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } MaterializedViewQuery other = (MaterializedViewQuery) o; return other.getQuery().equals(query); }
@Override public boolean doMonitor(ServiceEmitter emitter) { List<DataSourceOptimizerStats> stats = optimizer.getAndResetStats(); for (DataSourceOptimizerStats stat : stats) { final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder(); builder.setDimension("dataSource", stat.getBase()); emitter.emit(builder.build("/materialized/view/query/totalNum", stat.getTotalcount())); emitter.emit(builder.build("/materialized/view/query/hits", stat.getHitcount())); emitter.emit(builder.build("/materialized/view/query/hitRate", stat.getHitRate())); emitter.emit(builder.build("/materialized/view/select/avgCostMS", stat.getOptimizerCost())); Map<String, Long> derivativesStats = stat.getDerivativesHitCount(); for (String derivative : derivativesStats.keySet()) { builder.setDimension("derivative", derivative); emitter.emit(builder.build("/materialized/view/derivative/numSelected", derivativesStats.get(derivative))); } final ServiceMetricEvent.Builder builder2 = new ServiceMetricEvent.Builder(); builder2.setDimension("dataSource", stat.getBase()); for (Set<String> fields : stat.getMissFields().keySet()) { builder2.setDimension("fields", fields.toString()); emitter.emit(builder2.build("/materialized/view/missNum", stat.getMissFields().get(fields).get())); } } return true; } }
Set<DerivativeDataSource> derivatives = DerivativeDataSourceManager.getDerivatives(datasourceName); Set<String> requiredFields = MaterializedViewUtils.getRequiredFields(query); derivativesHitCount.putIfAbsent(derivativeDataSource.getName(), new AtomicLong(0)); if (derivativeDataSource.getColumns().containsAll(requiredFields)) { derivativesWithRequiredFields.add(derivativeDataSource); final List<Interval> derivativeIntervals = remainingQueryIntervals.stream() .flatMap(interval -> serverView .getTimeline((new TableDataSource(derivativeDataSource.getName()))) .lookup(interval) .stream() remainingQueryIntervals = MaterializedViewUtils.minus(remainingQueryIntervals, derivativeIntervals); queries.add( query.withDataSource(new TableDataSource(derivativeDataSource.getName())) .withQuerySegmentSpec(new MultipleIntervalSegmentSpec(derivativeIntervals)) ); derivativesHitCount.get(derivativeDataSource.getName()).incrementAndGet(); if (remainingQueryIntervals.isEmpty()) { break;
@Override public boolean equals(Object o) { if (o == null) { return false; } if (!(o instanceof DerivativeDataSource)) { return false; } DerivativeDataSource that = (DerivativeDataSource) o; return name.equals(that.getName()) && baseDataSource.equals(that.getBaseDataSource()) && columns.equals(that.getColumns()); }
Map<String, Set<DerivativeDataSource>> baseToDerivatives = DerivativeDataSourceManager.getAllDerivatives(); for (Map.Entry<String, Set<DerivativeDataSource>> entry : baseToDerivatives.entrySet()) { Map<String, Long> derivativesStat = new HashMap<>(); for (DerivativeDataSource derivative : entry.getValue()) { derivativesStat.put( derivative.getName(), derivativesHitCountSnapshot.getOrDefault(derivative.getName(), new AtomicLong(0)).get() ); new DataSourceOptimizerStats( entry.getKey(), hitCountSnapshot.getOrDefault(entry.getKey(), new AtomicLong(0)).get(),
@Override public Sequence run(QueryPlus queryPlus, Map responseContext) { Query realQuery = getRealQuery(queryPlus.getQuery()); QueryToolChest realQueryToolChest = warehouse.getToolChest(realQuery); QueryRunner realQueryRunner = realQueryToolChest.preMergeQueryDecoration( new MaterializedViewQueryRunner(runner, optimizer) ); return realQueryRunner.run(queryPlus.withQuery(realQuery), responseContext); } };
@Override public MaterializedViewQuery withQuerySegmentSpec(QuerySegmentSpec spec) { return new MaterializedViewQuery(query.withQuerySegmentSpec(spec), optimizer); }
@Override public Function makePreComputeManipulatorFn(Query query, MetricManipulationFn fn) { Query realQuery = getRealQuery(query); return warehouse.getToolChest(realQuery).makePreComputeManipulatorFn(realQuery, fn); }
.map( new ResultSetMapper<Object>()
@Override public Sequence<T> run(QueryPlus<T> queryPlus, Map<String, Object> responseContext) { Query query = queryPlus.getQuery(); return new MergeSequence<>( query.getResultOrdering(), Sequences.simple( Lists.transform( optimizer.optimize(query), new Function<Query, Sequence<T>>() { @Override public Sequence<T> apply(Query query) { return runner.run( queryPlus.withQuery(query), responseContext ); } } ) ) ); } }
dimensions.addAll(extractFieldsFromAggregations(q.getAggregatorSpecs())); dimensions.add(q.getDimensionSpec().getDimension()); } else if (query instanceof TimeseriesQuery) { TimeseriesQuery q = (TimeseriesQuery) query; dimensions.addAll(extractFieldsFromAggregations(q.getAggregatorSpecs())); } else if (query instanceof GroupByQuery) { GroupByQuery q = (GroupByQuery) query; dimensions.addAll(extractFieldsFromAggregations(q.getAggregatorSpecs())); for (DimensionSpec spec : q.getDimensions()) { String dim = spec.getDimension();
final Duration delay = config.getPollDuration().toStandardDuration(); future = exec.scheduleWithFixedDelay( new Runnable() {
@Override public MaterializedViewQuery withDataSource(DataSource dataSource) { return new MaterializedViewQuery(query.withDataSource(dataSource), optimizer); }
@Override public Function makePostComputeManipulatorFn(Query query, MetricManipulationFn fn) { Query realQuery = getRealQuery(query); return warehouse.getToolChest(realQuery).makePostComputeManipulatorFn(realQuery, fn); }
@Override public MaterializedViewQuery withId(String id) { return new MaterializedViewQuery(query.withId(id), optimizer); }
@Override public QueryMetrics makeMetrics(Query query) { Query realQuery = getRealQuery(query); return warehouse.getToolChest(realQuery).makeMetrics(realQuery); }
@Override public MaterializedViewQuery withOverriddenContext(Map<String, Object> contextOverride) { return new MaterializedViewQuery(query.withOverriddenContext(contextOverride), optimizer); }
@Override public Sequence run(QueryPlus queryPlus, Map responseContext) { Query realQuery = getRealQuery(queryPlus.getQuery()); return warehouse.getToolChest(realQuery).mergeResults(runner).run(queryPlus.withQuery(realQuery), responseContext); } };