private static void validateScoresArePositive(Version indexCreatedVersion, Similarity similarity) throws IOException { CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000); TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130); SimWeight simWeight = similarity.computeWeight(2f, collectionStats, termStats); FieldInvertState state = new FieldInvertState(indexCreatedVersion.luceneVersion.major, "some_field", 20, 20, 0, 50); // length = 20, no overlap final long norm = similarity.computeNorm(state); LeafReader reader = new SingleNormLeafReader(norm); SimScorer scorer = similarity.simScorer(simWeight, reader.getContext()); for (int freq = 1; freq <= 10; ++freq) { float score = scorer.score(0, freq); if (score < 0) { DEPRECATION_LOGGER.deprecated("Similarities should not return negative scores:\n" + scorer.explain(0, Explanation.match(freq, "term freq"))); break; } } }
private static void validateScoresDoNotDecreaseWithFreq(Version indexCreatedVersion, Similarity similarity) throws IOException { CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000); TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130); SimWeight simWeight = similarity.computeWeight(2f, collectionStats, termStats); FieldInvertState state = new FieldInvertState(indexCreatedVersion.luceneVersion.major, "some_field", 20, 20, 0, 50); // length = 20, no overlap final long norm = similarity.computeNorm(state); LeafReader reader = new SingleNormLeafReader(norm); SimScorer scorer = similarity.simScorer(simWeight, reader.getContext()); float previousScore = Float.NEGATIVE_INFINITY; for (int freq = 1; freq <= 10; ++freq) { float score = scorer.score(0, freq); if (score < previousScore) { DEPRECATION_LOGGER.deprecated("Similarity scores should not decrease when term frequency increases:\n" + scorer.explain(0, Explanation.match(freq - 1, "term freq")) + "\n" + scorer.explain(0, Explanation.match(freq, "term freq"))); break; } previousScore = score; } }
private static void validateScoresDoNotIncreaseWithNorm(Version indexCreatedVersion, Similarity similarity) throws IOException { CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000); TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130); SimWeight simWeight = similarity.computeWeight(2f, collectionStats, termStats); SimScorer previousScorer = null; long previousNorm = 0; float previousScore = Float.POSITIVE_INFINITY; for (int length = 1; length <= 10; ++length) { FieldInvertState state = new FieldInvertState(indexCreatedVersion.luceneVersion.major, "some_field", length, length, 0, 50); // length = 20, no overlap final long norm = similarity.computeNorm(state); if (Long.compareUnsigned(previousNorm, norm) > 0) { // esoteric similarity, skip this check break; } LeafReader reader = new SingleNormLeafReader(norm); SimScorer scorer = similarity.simScorer(simWeight, reader.getContext()); float score = scorer.score(0, 1); if (score > previousScore) { DEPRECATION_LOGGER.deprecated("Similarity scores should not increase when norm increases:\n" + previousScorer.explain(0, Explanation.match(1, "term freq")) + "\n" + scorer.explain(0, Explanation.match(1, "term freq"))); break; } previousScorer = scorer; previousScore = score; previousNorm = norm; } }
@Override public DocIdSet getDocIdSet(final LeafReaderContext context, final Bits acceptDocs) throws IOException { // get a private context that is used to rewrite, createWeight and score eventually final LeafReaderContext privateContext = context.reader().getContext(); final Weight weight = new IndexSearcher(privateContext).createNormalizedWeight(query, false); DocIdSet set = new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { Scorer s = weight.scorer(privateContext); return s == null ? null : s.iterator(); } @Override public long ramBytesUsed() { return 0L; } }; return BitsFilteredDocIdSet.wrap(set, acceptDocs); }
@Override public BinaryDocValues getBinaryDocValues(String field) throws IOException { if (facetFields.contains(field)) { final OrdinalsReader ordsReader = getOrdinalsReader(field); return new OrdinalMappingBinaryDocValues(ordsReader.getReader(in.getContext())); } else { return in.getBinaryDocValues(field); } }
@Override public DocIdSet getDocIdSet(final LeafReaderContext context, final Bits acceptDocs) throws IOException { // get a private context that is used to rewrite, createWeight and score eventually final LeafReaderContext privateContext = context.reader().getContext(); final Weight weight = new IndexSearcher(privateContext).createNormalizedWeight(query, false); DocIdSet set = new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { Scorer s = weight.scorer(privateContext); return s == null ? null : s.iterator(); } @Override public long ramBytesUsed() { return 0L; } }; return BitsFilteredDocIdSet.wrap(set, acceptDocs); }
@Override public BinaryDocValues getBinaryDocValues(String field) throws IOException { if (facetFields.contains(field)) { final OrdinalsReader ordsReader = getOrdinalsReader(field); return new OrdinalMappingBinaryDocValues(ordsReader.getReader(in.getContext()), in.getBinaryDocValues(field)); } else { return in.getBinaryDocValues(field); } }
Spans spans = weight.getSpans(scrw.getContext(), Postings.OFFSETS); hitsFound = false; while(spans != null && spans.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
public TokenRangesLeafReader(DirectoryReader directoryReader, LeafReader in, Query query, TokenRangesBitsetFilterCache cache) throws IOException { super(in); try { in.addCoreClosedListener(cache); ElasticsearchDirectoryReader.addReaderCloseListener(directoryReader, cache); //in.addReaderClosedListener(cache); this.mask = cache.getBitSet(query, in.getContext()); if (mask == null) { numDocs = 0; hasDeletions = true; noBitMatch = new Bits.MatchNoBits(in.maxDoc()); } else { numDocs = mask.cardinality(); hasDeletions = numDocs < in.numDocs() || in.hasDeletions(); noBitMatch = null; } } catch (ExecutionException e) { throw new IOException(e); } }
comparators[i] = fields[i].getComparator(1, i).getLeafComparator(reader.getContext()); comparators[i].setScorer(FAKESCORER);
comparators[i] = fields[i].getComparator(1, i).getLeafComparator(reader.getContext()); comparators[i].setScorer(FAKESCORER);
private static void validateScoresArePositive(Version indexCreatedVersion, Similarity similarity) throws IOException { CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000); TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130); SimWeight simWeight = similarity.computeWeight(2f, collectionStats, termStats); FieldInvertState state = new FieldInvertState(indexCreatedVersion.luceneVersion.major, "some_field", 20, 20, 0, 50); // length = 20, no overlap final long norm = similarity.computeNorm(state); LeafReader reader = new SingleNormLeafReader(norm); SimScorer scorer = similarity.simScorer(simWeight, reader.getContext()); for (int freq = 1; freq <= 10; ++freq) { float score = scorer.score(0, freq); if (score < 0) { DEPRECATION_LOGGER.deprecated("Similarities should not return negative scores:\n" + scorer.explain(0, Explanation.match(freq, "term freq"))); break; } } }
private static void validateScoresDoNotDecreaseWithFreq(Version indexCreatedVersion, Similarity similarity) throws IOException { CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000); TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130); SimWeight simWeight = similarity.computeWeight(2f, collectionStats, termStats); FieldInvertState state = new FieldInvertState(indexCreatedVersion.luceneVersion.major, "some_field", 20, 20, 0, 50); // length = 20, no overlap final long norm = similarity.computeNorm(state); LeafReader reader = new SingleNormLeafReader(norm); SimScorer scorer = similarity.simScorer(simWeight, reader.getContext()); float previousScore = Float.NEGATIVE_INFINITY; for (int freq = 1; freq <= 10; ++freq) { float score = scorer.score(0, freq); if (score < previousScore) { DEPRECATION_LOGGER.deprecated("Similarity scores should not decrease when term frequency increases:\n" + scorer.explain(0, Explanation.match(freq - 1, "term freq")) + "\n" + scorer.explain(0, Explanation.match(freq, "term freq"))); break; } previousScore = score; } }
return internalReader.getContext();
private static void validateScoresDoNotIncreaseWithNorm(Version indexCreatedVersion, Similarity similarity) throws IOException { CollectionStatistics collectionStats = new CollectionStatistics("some_field", 1200, 1100, 3000, 2000); TermStatistics termStats = new TermStatistics(new BytesRef("some_value"), 100, 130); SimWeight simWeight = similarity.computeWeight(2f, collectionStats, termStats); SimScorer previousScorer = null; long previousNorm = 0; float previousScore = Float.POSITIVE_INFINITY; for (int length = 1; length <= 10; ++length) { FieldInvertState state = new FieldInvertState(indexCreatedVersion.luceneVersion.major, "some_field", length, length, 0, 50); // length = 20, no overlap final long norm = similarity.computeNorm(state); if (Long.compareUnsigned(previousNorm, norm) > 0) { // esoteric similarity, skip this check break; } LeafReader reader = new SingleNormLeafReader(norm); SimScorer scorer = similarity.simScorer(simWeight, reader.getContext()); float score = scorer.score(0, 1); if (score > previousScore) { DEPRECATION_LOGGER.deprecated("Similarity scores should not increase when norm increases:\n" + previousScorer.explain(0, Explanation.match(1, "term freq")) + "\n" + scorer.explain(0, Explanation.match(1, "term freq"))); break; } previousScorer = scorer; previousScore = score; previousNorm = norm; } }