final IndexReader reader = context.searcher().getIndexReader(); final RandomAccessOrds values = reader.leaves().size() > 0 ? valuesSource.globalOrdinalsValues(context.searcher().getIndexReader().leaves().get(0)) : DocValues.emptySortedSet(); this.valueCount = values.getValueCount(); this.lookupGlobalOrd = values::lookupOrd;
ValuesSource.Bytes.WithOrdinals valueSourceWithOrdinals = (ValuesSource.Bytes.WithOrdinals) valuesSource; IndexSearcher indexSearcher = context.searcher(); maxOrd = valueSourceWithOrdinals.globalMaxOrd(indexSearcher); ratio = maxOrd / ((double) indexSearcher.getIndexReader().numDocs()); } else {
ValuesSource.Bytes.WithOrdinals valueSourceWithOrdinals = (ValuesSource.Bytes.WithOrdinals) valuesSource; IndexSearcher indexSearcher = aggregationContext.searchContext().searcher(); maxOrd = valueSourceWithOrdinals.globalMaxOrd(indexSearcher); ratio = maxOrd / ((double) indexSearcher.getIndexReader().numDocs()); } else {
final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx);
mapSegmentCountsToGlobalCounts(mapping); final SortedSetDocValues segmentOrds = valuesSource.ordinalsValues(ctx); segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + segmentOrds.getValueCount()); assert sub == LeafBucketCollector.NO_OP_COLLECTOR; final SortedDocValues singleValues = DocValues.unwrapSingleton(segmentOrds); mapping = valuesSource.globalOrdinalsMapping(ctx); if (singleValues != null) { return new LeafBucketCollectorBase(sub, segmentOrds) {
mapSegmentCountsToGlobalCounts(); globalOrds = valuesSource.globalOrdinalsValues(ctx); segmentOrds = valuesSource.ordinalsValues(ctx); segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + segmentOrds.getValueCount()); assert sub == LeafBucketCollector.NO_OP_COLLECTOR;
mapSegmentCountsToGlobalCounts(mapping); final SortedSetDocValues segmentOrds = valuesSource.ordinalsValues(ctx); segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + segmentOrds.getValueCount()); assert sub == LeafBucketCollector.NO_OP_COLLECTOR; final SortedDocValues singleValues = DocValues.unwrapSingleton(segmentOrds); mapping = valuesSource.globalOrdinalsMapping(ctx); if (singleValues != null) { return new LeafBucketCollectorBase(sub, segmentOrds) {
return LeafBucketCollector.NO_OP_COLLECTOR; final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx); final Bits parentDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), parentFilter.scorerSupplier(ctx)); return new LeafBucketCollector() {
@Override protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { if (valuesSource == null) { return LeafBucketCollector.NO_OP_COLLECTOR; } if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { SortedSetDocValues values = ((ValuesSource.Bytes.WithOrdinals) valuesSource).ordinalsValues(ctx); return new SortedSetRangeLeafCollector(values, ranges, sub) { @Override protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException { collectBucket(sub, doc, bucket); } }; } else { SortedBinaryDocValues values = valuesSource.bytesValues(ctx); return new SortedBinaryRangeLeafCollector(values, ranges, sub) { @Override protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException { collectBucket(sub, doc, bucket); } }; } }
@Override protected final void doPostCollection() throws IOException { IndexReader indexReader = context().searcher().getIndexReader(); for (LeafReaderContext ctx : indexReader.leaves()) { Scorer childDocsScorer = outFilter.scorer(ctx); if (childDocsScorer == null) { continue; } DocIdSetIterator childDocsIter = childDocsScorer.iterator(); final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx); final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx); // Set the scorer, since we now replay only the child docIds sub.setScorer(new ConstantScoreScorer(null, 1f, childDocsIter)); final Bits liveDocs = ctx.reader().getLiveDocs(); for (int docId = childDocsIter.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = childDocsIter.nextDoc()) { if (liveDocs != null && liveDocs.get(docId) == false) { continue; } if (globalOrdinals.advanceExact(docId)) { int globalOrdinal = (int) globalOrdinals.nextOrd(); assert globalOrdinal != -1 && globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS; if (existsGlobalOrdinal(globalOrdinal)) { collectBucket(sub, docId, 0); } } } } }
private Collector pickCollector(LeafReaderContext ctx) throws IOException { if (valuesSource == null) { return new EmptyCollector(); } if (valuesSource instanceof ValuesSource.Numeric) { ValuesSource.Numeric source = (ValuesSource.Numeric) valuesSource; MurmurHash3Values hashValues = source.isFloatingPoint() ? MurmurHash3Values.hash(source.doubleValues(ctx)) : MurmurHash3Values.hash(source.longValues(ctx)); return new DirectCollector(counts, hashValues); } if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { ValuesSource.Bytes.WithOrdinals source = (ValuesSource.Bytes.WithOrdinals) valuesSource; final SortedSetDocValues ordinalValues = source.ordinalsValues(ctx); final long maxOrd = ordinalValues.getValueCount(); if (maxOrd == 0) { return new EmptyCollector(); } final long ordinalsMemoryUsage = OrdinalsCollector.memoryOverhead(maxOrd); final long countsMemoryUsage = HyperLogLogPlusPlus.memoryUsage(precision); // only use ordinals if they don't increase memory usage by more than 25% if (ordinalsMemoryUsage < countsMemoryUsage / 4) { return new OrdinalsCollector(counts, ordinalValues, context.bigArrays()); } } return new DirectCollector(counts, MurmurHash3Values.hash(valuesSource.bytesValues(ctx))); }
private Collector pickCollector(LeafReaderContext ctx) throws IOException { if (valuesSource == null) { return new EmptyCollector(); } if (valuesSource instanceof ValuesSource.Numeric) { ValuesSource.Numeric source = (ValuesSource.Numeric) valuesSource; MurmurHash3Values hashValues = source.isFloatingPoint() ? MurmurHash3Values.hash(source.doubleValues(ctx)) : MurmurHash3Values.hash(source.longValues(ctx)); return new DirectCollector(counts, hashValues); } if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { ValuesSource.Bytes.WithOrdinals source = (ValuesSource.Bytes.WithOrdinals) valuesSource; final SortedSetDocValues ordinalValues = source.ordinalsValues(ctx); final long maxOrd = ordinalValues.getValueCount(); if (maxOrd == 0) { return new EmptyCollector(); } final long ordinalsMemoryUsage = OrdinalsCollector.memoryOverhead(maxOrd); final long countsMemoryUsage = HyperLogLogPlusPlus.memoryUsage(precision); // only use ordinals if they don't increase memory usage by more than 25% if (ordinalsMemoryUsage < countsMemoryUsage / 4) { return new OrdinalsCollector(counts, ordinalValues, context.bigArrays()); } } return new DirectCollector(counts, MurmurHash3Values.hash(valuesSource.bytesValues(ctx))); }
public GlobalOrdinalsStringTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource, BucketOrder order, DocValueFormat format, BucketCountThresholds bucketCountThresholds, IncludeExclude.OrdinalsFilter includeExclude, SearchContext context, Aggregator parent, boolean remapGlobalOrds, SubAggCollectionMode collectionMode, boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException { super(name, factories, context, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError, pipelineAggregators, metaData); this.valuesSource = valuesSource; this.includeExclude = includeExclude; final IndexReader reader = context.searcher().getIndexReader(); final SortedSetDocValues values = reader.leaves().size() > 0 ? valuesSource.globalOrdinalsValues(context.searcher().getIndexReader().leaves().get(0)) : DocValues.emptySortedSet(); this.valueCount = values.getValueCount(); this.lookupGlobalOrd = values::lookupOrd; this.acceptedGlobalOrdinals = includeExclude != null ? includeExclude.acceptedGlobalOrdinals(values) : null; this.bucketOrds = remapGlobalOrds ? new LongHash(1, context.bigArrays()) : null; }
@Override protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { if (valuesSource == null) { return LeafBucketCollector.NO_OP_COLLECTOR; } if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { SortedSetDocValues values = ((ValuesSource.Bytes.WithOrdinals) valuesSource).ordinalsValues(ctx); return new SortedSetRangeLeafCollector(values, ranges, sub) { @Override protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException { collectBucket(sub, doc, bucket); } }; } else { SortedBinaryDocValues values = valuesSource.bytesValues(ctx); return new SortedBinaryRangeLeafCollector(values, ranges, sub) { @Override protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException { collectBucket(sub, doc, bucket); } }; } }
@Override protected final void doPostCollection() throws IOException { IndexReader indexReader = context().searcher().getIndexReader(); for (LeafReaderContext ctx : indexReader.leaves()) { Scorer childDocsScorer = outFilter.scorer(ctx); if (childDocsScorer == null) { continue; } DocIdSetIterator childDocsIter = childDocsScorer.iterator(); final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx); final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx); // Set the scorer, since we now replay only the child docIds sub.setScorer(new ConstantScoreScorer(null, 1f, childDocsIter)); final Bits liveDocs = ctx.reader().getLiveDocs(); for (int docId = childDocsIter.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = childDocsIter.nextDoc()) { if (liveDocs != null && liveDocs.get(docId) == false) { continue; } if (globalOrdinals.advanceExact(docId)) { int globalOrdinal = (int) globalOrdinals.nextOrd(); assert globalOrdinal != -1 && globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS; if (existsGlobalOrdinal(globalOrdinal)) { collectBucket(sub, docId, 0); } } } } }
@Override Collector getLeafCollector(LeafReaderContext context, Collector next) throws IOException { final SortedSetDocValues dvs = vs.globalOrdinalsValues(context); if (lookup == null) { lookup = dvs; if (topValue != null && topValueGlobalOrd == null) { topValueGlobalOrd = lookup.lookupTerm(topValue); if (topValueGlobalOrd < 0) { // convert negative insert position topValueGlobalOrd = -topValueGlobalOrd - 1; isTopValueInsertionPoint = true; } } } return doc -> { if (dvs.advanceExact(doc)) { long ord; while ((ord = dvs.nextOrd()) != NO_MORE_ORDS) { values[0] = ord; next.collect(doc); } } }; } }
private Collector pickCollector(LeafReaderContext ctx) throws IOException { if (valuesSource == null) { return new EmptyCollector(); } if (valuesSource instanceof ValuesSource.Numeric) { ValuesSource.Numeric source = (ValuesSource.Numeric) valuesSource; MurmurHash3Values hashValues = source.isFloatingPoint() ? MurmurHash3Values.hash(source.doubleValues(ctx)) : MurmurHash3Values.hash(source.longValues(ctx)); return new DirectCollector(counts, hashValues); } if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { ValuesSource.Bytes.WithOrdinals source = (ValuesSource.Bytes.WithOrdinals) valuesSource; final RandomAccessOrds ordinalValues = source.ordinalsValues(ctx); final long maxOrd = ordinalValues.getValueCount(); if (maxOrd == 0) { return new EmptyCollector(); } final long ordinalsMemoryUsage = OrdinalsCollector.memoryOverhead(maxOrd); final long countsMemoryUsage = HyperLogLogPlusPlus.memoryUsage(precision); // only use ordinals if they don't increase memory usage by more than 25% if (ordinalsMemoryUsage < countsMemoryUsage / 4) { return new OrdinalsCollector(counts, ordinalValues, context.bigArrays()); } } return new DirectCollector(counts, MurmurHash3Values.hash(valuesSource.bytesValues(ctx))); }
@Override protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { if (valuesSource == null) { return LeafBucketCollector.NO_OP_COLLECTOR; } if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { SortedSetDocValues values = ((ValuesSource.Bytes.WithOrdinals) valuesSource).ordinalsValues(ctx); return new SortedSetRangeLeafCollector(values, ranges, sub) { @Override protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException { collectBucket(sub, doc, bucket); } }; } else { SortedBinaryDocValues values = valuesSource.bytesValues(ctx); return new SortedBinaryRangeLeafCollector(values, ranges, sub) { @Override protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException { collectBucket(sub, doc, bucket); } }; } }
public GlobalOrdinalsStringTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource, BucketOrder order, DocValueFormat format, BucketCountThresholds bucketCountThresholds, IncludeExclude.OrdinalsFilter includeExclude, SearchContext context, Aggregator parent, boolean remapGlobalOrds, SubAggCollectionMode collectionMode, boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException { super(name, factories, context, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError, pipelineAggregators, metaData); this.valuesSource = valuesSource; this.includeExclude = includeExclude; final IndexReader reader = context.searcher().getIndexReader(); final SortedSetDocValues values = reader.leaves().size() > 0 ? valuesSource.globalOrdinalsValues(context.searcher().getIndexReader().leaves().get(0)) : DocValues.emptySortedSet(); this.valueCount = values.getValueCount(); this.lookupGlobalOrd = values::lookupOrd; this.acceptedGlobalOrdinals = includeExclude != null ? includeExclude.acceptedGlobalOrdinals(values) : null; this.bucketOrds = remapGlobalOrds ? new LongHash(1, context.bigArrays()) : null; }
private Collector pickCollector(LeafReaderContext ctx) throws IOException { if (valuesSource == null) { return new EmptyCollector(); } if (valuesSource instanceof ValuesSource.Numeric) { ValuesSource.Numeric source = (ValuesSource.Numeric) valuesSource; MurmurHash3Values hashValues = source.isFloatingPoint() ? MurmurHash3Values.hash(source.doubleValues(ctx)) : MurmurHash3Values.hash(source.longValues(ctx)); return new DirectCollector(counts, hashValues); } if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { ValuesSource.Bytes.WithOrdinals source = (ValuesSource.Bytes.WithOrdinals) valuesSource; final RandomAccessOrds ordinalValues = source.ordinalsValues(ctx); final long maxOrd = ordinalValues.getValueCount(); if (maxOrd == 0) { return new EmptyCollector(); } final long ordinalsMemoryUsage = OrdinalsCollector.memoryOverhead(maxOrd); final long countsMemoryUsage = HyperLogLogPlusPlus.memoryUsage(precision); // only use ordinals if they don't increase memory usage by more than 25% if (ordinalsMemoryUsage < countsMemoryUsage / 4) { return new OrdinalsCollector(counts, ordinalValues, context.bigArrays()); } } return new DirectCollector(counts, MurmurHash3Values.hash(valuesSource.bytesValues(ctx))); }