@Override public LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) throws IOException { final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); final OrdinalMap map = global.getOrdinalMap(); if (map == null) { // segments and global ordinals are the same return LongUnaryOperator.identity(); } final org.apache.lucene.util.LongValues segmentToGlobalOrd = map.getGlobalOrds(context.ord); return segmentToGlobalOrd::get; } }
@Override public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) { final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); final AtomicOrdinalsFieldData atomicFieldData = global.load(context); return atomicFieldData.getOrdinalsValues(); }
/** * Build global ordinals for the provided {@link IndexReader}. */ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, IndexSettings indexSettings, CircuitBreakerService breakerService, Logger logger, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i)); subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); final long memorySizeInBytes = ordinalMap.ramBytesUsed(); breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(memorySizeInBytes); if (logger.isDebugEnabled()) { logger.debug( "global-ordinals [{}][{}] took [{}]", indexFieldData.getFieldName(), ordinalMap.getValueCount(), new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS) ); } return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, memorySizeInBytes, scriptFunction ); }
@Override public SortedSetDocValues ordinalsValues(LeafReaderContext context) { final AtomicOrdinalsFieldData atomicFieldData = indexFieldData.load(context); return atomicFieldData.getOrdinalsValues(); }
@Override public MultiDocValues.OrdinalMap getOrdinalMap() { return idFieldData.getOrdinalMap(); }
/** * Build global ordinals for the provided {@link IndexReader}. */ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, Settings settings, CircuitBreakerService breakerService, ESLogger logger) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i)); subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); final long memorySizeInBytes = ordinalMap.ramBytesUsed(); breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(memorySizeInBytes); if (logger.isDebugEnabled()) { logger.debug( "Global-ordinals[{}][{}] took {} ms", indexFieldData.getFieldNames().fullName(), ordinalMap.getValueCount(), TimeValue.nsecToMSec(System.nanoTime() - startTimeNS) ); } return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings, indexFieldData.getFieldNames(), indexFieldData.getFieldDataType(), atomicFD, ordinalMap, memorySizeInBytes ); }
public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData) throws IOException { assert indexReader.leaves().size() > 1; final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = new AbstractAtomicOrdinalsFieldData(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) { @Override public SortedSetDocValues getOrdinalsValues() { return DocValues.emptySortedSet(); } @Override public long ramBytesUsed() { return 0; } @Override public Collection<Accountable> getChildResources() { return Collections.emptyList(); } @Override public void close() { } }; subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, 0, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION ); }
@Override public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { return new UidIndexFieldData(index, type, idFieldData.loadGlobal(indexReader)); }
public static IndexOrdinalsFieldData buildEmpty(Index index, Settings settings, final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData) throws IOException { assert indexReader.leaves().size() > 1; final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = new AbstractAtomicOrdinalsFieldData() { @Override public RandomAccessOrds getOrdinalsValues() { return DocValues.emptySortedSet(); } @Override public long ramBytesUsed() { return 0; } @Override public Collection<Accountable> getChildResources() { return Collections.emptyList(); } @Override public void close() { } }; subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); return new InternalGlobalOrdinalsIndexFieldData(index, settings, indexFieldData.getFieldNames(), indexFieldData.getFieldDataType(), atomicFD, ordinalMap, 0 ); }
@Override public void clear() { idFieldData.clear(); }
@Override public SortedBinaryDocValues bytesValues(LeafReaderContext context) { final AtomicOrdinalsFieldData atomicFieldData = indexFieldData.load(context); return atomicFieldData.getBytesValues(); }
@Override public Query rewrite(IndexReader reader) throws IOException { Query rewritten = super.rewrite(reader); if (rewritten != this) { return rewritten; } if (reader instanceof DirectoryReader) { IndexSearcher indexSearcher = new IndexSearcher(reader); indexSearcher.setQueryCache(null); indexSearcher.setSimilarity(similarity); IndexOrdinalsFieldData indexParentChildFieldData = fieldDataJoin.loadGlobal((DirectoryReader) reader); OrdinalMap ordinalMap = indexParentChildFieldData.getOrdinalMap(); return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode, ordinalMap, minChildren, maxChildren); } else { if (reader.leaves().isEmpty() && reader.numDocs() == 0) { // asserting reader passes down a MultiReader during rewrite which makes this // blow up since for this query to work we have to have a DirectoryReader otherwise // we can't load global ordinals - for this to work we simply check if the reader has no leaves // and rewrite to match nothing return new MatchNoDocsQuery(); } throw new IllegalStateException("can't load global ordinals for reader of type: " + reader.getClass() + " must be a DirectoryReader"); } }
public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData) throws IOException { assert indexReader.leaves().size() > 1; final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = new AbstractAtomicOrdinalsFieldData(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) { @Override public SortedSetDocValues getOrdinalsValues() { return DocValues.emptySortedSet(); } @Override public long ramBytesUsed() { return 0; } @Override public Collection<Accountable> getChildResources() { return Collections.emptyList(); } @Override public void close() { } }; subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, 0, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION ); }
@Override public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) { final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); final AtomicOrdinalsFieldData atomicFieldData = global.load(context); return atomicFieldData.getOrdinalsValues(); } }
@Override public LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) throws IOException { final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); final OrdinalMap map = global.getOrdinalMap(); if (map == null) { // segments and global ordinals are the same return LongUnaryOperator.identity(); } final org.apache.lucene.util.LongValues segmentToGlobalOrd = map.getGlobalOrds(context.ord); return segmentToGlobalOrd::get; } }
@Override protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException { final SortedSetDocValues values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues(); final SortedDocValues selectedValues; if (nested == null) { selectedValues = sortMode.select(values); } else { final BitSet rootDocs = nested.rootDocs(context); final DocIdSetIterator innerDocs = nested.innerDocs(context); final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE; selectedValues = sortMode.select(values, rootDocs, innerDocs, maxChildren); } if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) { return selectedValues; } else { return new ReplaceMissing(selectedValues, missingBytes); } }
/** * Build global ordinals for the provided {@link IndexReader}. */ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, IndexSettings indexSettings, CircuitBreakerService breakerService, Logger logger, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i)); subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); final long memorySizeInBytes = ordinalMap.ramBytesUsed(); breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(memorySizeInBytes); if (logger.isDebugEnabled()) { logger.debug( "global-ordinals [{}][{}] took [{}]", indexFieldData.getFieldName(), ordinalMap.getValueCount(), new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS) ); } return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, memorySizeInBytes, scriptFunction ); }
@Override public Query rewrite(IndexReader reader) throws IOException { Query rewritten = super.rewrite(reader); if (rewritten != this) { return rewritten; } if (reader instanceof DirectoryReader) { IndexSearcher indexSearcher = new IndexSearcher(reader); indexSearcher.setQueryCache(null); indexSearcher.setSimilarity(similarity); IndexOrdinalsFieldData indexParentChildFieldData = fieldDataJoin.loadGlobal((DirectoryReader) reader); OrdinalMap ordinalMap = indexParentChildFieldData.getOrdinalMap(); return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode, ordinalMap, minChildren, maxChildren); } else { if (reader.leaves().isEmpty() && reader.numDocs() == 0) { // asserting reader passes down a MultiReader during rewrite which makes this // blow up since for this query to work we have to have a DirectoryReader otherwise // we can't load global ordinals - for this to work we simply check if the reader has no leaves // and rewrite to match nothing return new MatchNoDocsQuery(); } throw new IllegalStateException("can't load global ordinals for reader of type: " + reader.getClass() + " must be a DirectoryReader"); } }
public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData) throws IOException { assert indexReader.leaves().size() > 1; final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = new AbstractAtomicOrdinalsFieldData(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) { @Override public RandomAccessOrds getOrdinalsValues() { return DocValues.emptySortedSet(); } @Override public long ramBytesUsed() { return 0; } @Override public Collection<Accountable> getChildResources() { return Collections.emptyList(); } @Override public void close() { } }; subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, 0, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION ); }
@Override public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) { final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); final AtomicOrdinalsFieldData atomicFieldData = global.load(context); return atomicFieldData.getOrdinalsValues(); } }