@Override public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsumer dvConsumer) throws IOException { final PackedLongValues values; if (finalValues == null) { values = pending.build(); } else { values = finalValues; } final SortingLeafReader.CachedNumericDVs sorted; if (sortMap != null) { NumericDocValues oldValues = new BufferedNumericDocValues(values, docsWithField.iterator()); sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap, oldValues); } else { sorted = null; } dvConsumer.addNumericField(fieldInfo, new EmptyDocValuesProducer() { @Override public NumericDocValues getNumeric(FieldInfo fieldInfo) { if (fieldInfo != NumericDocValuesWriter.this.fieldInfo) { throw new IllegalArgumentException("wrong fieldInfo"); } if (sorted == null) { return new BufferedNumericDocValues(values, docsWithField.iterator()); } else { return new SortingLeafReader.SortingNumericDocValues(sorted); } } }); }
public void flush(SegmentWriteState state, Sorter.DocMap sortMap, NormsConsumer normsConsumer) throws IOException { final PackedLongValues values = pending.build(); final SortingLeafReader.CachedNumericDVs sorted; if (sortMap != null) {
final PackedLongValues remapped = builders[i].build(); final Bits liveDocs = readers.get(i).getLiveDocs(); docMaps[i] = new MergeState.DocMap() {
public void flush(SegmentWriteState state, NormsConsumer normsConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); final PackedLongValues values = pending.build(); normsConsumer.addNormsField(fieldInfo, new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new NumericIterator(maxDoc, values); } }); }
final Ordinals ordinals = builder.build(); data = new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffset.build(), ordinals); success = true; return data;
public void mergeBuckets(long[] mergeMap) { List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry sourceEntry : entries) { PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build())); } entries = newEntries; // if there are buckets that have been collected in the current segment // we need to update the bucket ordinals there too if (buckets.size() > 0) { PackedLongValues currentBuckets = buckets.build(); PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } buckets = newBuckets; } }
public MultiOrdinals(OrdinalsBuilder builder, float acceptableOverheadRatio) { multiValued = builder.getNumMultiValuesDocs() > 0; valueCount = builder.getValueCount(); PackedLongValues.Builder endOffsetsBuilder = PackedLongValues.monotonicBuilder(OFFSETS_PAGE_SIZE, acceptableOverheadRatio); PackedLongValues.Builder ordsBuilder = PackedLongValues.packedBuilder(OFFSETS_PAGE_SIZE, acceptableOverheadRatio); long lastEndOffset = 0; for (int i = 0; i < builder.maxDoc(); ++i) { final LongsRef docOrds = builder.docOrds(i); final long endOffset = lastEndOffset + docOrds.length; endOffsetsBuilder.add(endOffset); for (int j = 0; j < docOrds.length; ++j) { ordsBuilder.add(docOrds.longs[docOrds.offset + j]); } lastEndOffset = endOffset; } endOffsets = endOffsetsBuilder.build(); ords = ordsBuilder.build(); assert endOffsets.size() == builder.maxDoc(); assert ords.size() == builder.getTotalNumOrds() : ords.size() + " != " + builder.getTotalNumOrds(); }
if (finalOrds == null) { sortedValues = hash.sort(); ords = pending.build(); ordMap = new int[valueCount]; for (int ord = 0; ord < valueCount; ord++) {
@Override public void flush(SegmentWriteState state, Sorter.DocMap sortMap, DocValuesConsumer dvConsumer) throws IOException { bytes.freeze(false); final PackedLongValues lengths = this.lengths.build(); final SortingLeafReader.CachedBinaryDVs sorted; if (sortMap != null) { sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap, new BufferedBinaryDocValues(lengths, maxLength, bytes.getDataInput(), docsWithField.iterator())); } else { sorted = null; } dvConsumer.addBinaryField(fieldInfo, new EmptyDocValuesProducer() { @Override public BinaryDocValues getBinary(FieldInfo fieldInfoIn) { if (fieldInfoIn != fieldInfo) { throw new IllegalArgumentException("wrong fieldInfo"); } if (sorted == null) { return new BufferedBinaryDocValues(lengths, maxLength, bytes.getDataInput(), docsWithField.iterator()); } else { return new SortingLeafReader.SortingBinaryDocValues(sorted); } } }); }
@Override Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException { assert sortField instanceof SortedSetSortField; assert finalOrds == null && finalOrdCounts == null && finalSortedValues == null && finalOrdMap == null; int valueCount = hash.size(); finalOrds = pending.build(); finalOrdCounts = pendingCounts.build(); finalSortedValues = hash.sort(); finalOrdMap = new int[valueCount]; for (int ord = 0; ord < valueCount; ord++) { finalOrdMap[finalSortedValues[ord]] = ord; } SortedSetSortField sf = (SortedSetSortField) sortField; final SortedSetDocValues dvs = new BufferedSortedSetDocValues(finalSortedValues, finalOrdMap, hash, finalOrds, finalOrdCounts, maxCount, docsWithField.iterator()); return Sorter.getDocComparator(maxDoc, sf, () -> SortedSetSelector.wrap(dvs, sf.getSelector()), () -> null); }
@Override Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException { assert sortField.getType().equals(SortField.Type.STRING); assert finalSortedValues == null && finalOrdMap == null &&finalOrds == null; int valueCount = hash.size(); finalSortedValues = hash.sort(); finalOrds = pending.build(); finalOrdMap = new int[valueCount]; for (int ord = 0; ord < valueCount; ord++) { finalOrdMap[finalSortedValues[ord]] = ord; } final SortedDocValues docValues = new BufferedSortedDocValues(hash, valueCount, finalOrds, finalSortedValues, finalOrdMap, docsWithField.iterator()); return Sorter.getDocComparator(maxDoc, sortField, () -> docValues, () -> null); }
@Override Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException { assert finalValues == null; finalValues = pending.build(); final BufferedNumericDocValues docValues = new BufferedNumericDocValues(finalValues, docsWithField.iterator()); return Sorter.getDocComparator(maxDoc, sortField, () -> null, () -> docValues); }
static PackedLongValues removeDeletes(final int maxDoc, final Bits liveDocs) { final PackedLongValues.Builder docMapBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); int del = 0; for (int i = 0; i < maxDoc; ++i) { docMapBuilder.add(i - del); if (liveDocs.get(i) == false) { ++del; } } return docMapBuilder.build(); } }
@Override Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException { assert sortField instanceof SortedNumericSortField; assert finalValues == null && finalValuesCount == null; finalValues = pending.build(); finalValuesCount = pendingCounts.build(); final SortedNumericDocValues docValues = new BufferedSortedNumericDocValues(finalValues, finalValuesCount, docsWithField.iterator()); SortedNumericSortField sf = (SortedNumericSortField) sortField; return Sorter.getDocComparator(maxDoc, sf, () -> null, () -> SortedNumericSelector.wrap(docValues, sf.getSelector(), sf.getNumericType())); }