/** @see #packedBuilder(int, float) */ public static PackedLongValues.Builder packedBuilder(float acceptableOverheadRatio) { return packedBuilder(DEFAULT_PAGE_SIZE, acceptableOverheadRatio); }
@Override public long get(long ord) { return ord + deltas.get(ord); } };
public BufferedSortedDocValues(BytesRefHash hash, int valueCount, PackedLongValues docToOrd, int[] sortedValues, int[] ordMap, DocIdSetIterator docsWithField) { this.hash = hash; this.valueCount = valueCount; this.sortedValues = sortedValues; this.iter = docToOrd.iterator(); this.ordMap = ordMap; this.docsWithField = docsWithField; }
PackedLongValues.Builder globalOrdDeltas = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); PackedLongValues.Builder firstSegments = PackedLongValues.packedBuilder(PackedInts.COMPACT); final PackedLongValues.Builder[] ordDeltas = new PackedLongValues.Builder[subs.length]; for (int i = 0; i < ordDeltas.length; i++) { ordDeltas[i] = PackedLongValues.monotonicBuilder(acceptableOverheadRatio); long ramBytesUsed = BASE_RAM_BYTES_USED + this.globalOrdDeltas.ramBytesUsed() + this.firstSegments.ramBytesUsed() + RamUsageEstimator.shallowSizeOf(segmentToGlobalOrds) + segmentMap.ramBytesUsed(); for (int i = 0; i < ordDeltas.length; ++i) { } else { final int bitsRequired = ordDeltaBits[i] < 0 ? 64 : PackedInts.bitsRequired(ordDeltaBits[i]); final long monotonicBits = deltas.ramBytesUsed() * 8; final long packedBits = bitsRequired * deltas.size(); if (deltas.size() <= Integer.MAX_VALUE && packedBits <= monotonicBits * (1 + acceptableOverheadRatio)) { final int size = (int) deltas.size(); final PackedInts.Mutable newDeltas = PackedInts.getMutable(size, bitsRequired, acceptableOverheadRatio); final PackedLongValues.Iterator it = deltas.iterator(); for (int ord = 0; ord < size; ++ord) { newDeltas.set(ord, it.next()); ramBytesUsed += deltas.ramBytesUsed();
final LeafBucketCollector leafCollector = collector.getLeafCollector(entry.context); DocIdSetIterator docIt = null; if (needsScores && entry.docDeltas.size() > 0) { Scorer scorer = weight.scorer(entry.context); leafCollector.setScorer(scorer); final PackedLongValues.Iterator docDeltaIterator = entry.docDeltas.iterator(); final PackedLongValues.Iterator buckets = entry.buckets.iterator(); int doc = 0; for (long i = 0, end = entry.docDeltas.size(); i < end; ++i) { doc += docDeltaIterator.next(); final long bucket = buckets.next();
public MultiOrdinals(OrdinalsBuilder builder, float acceptableOverheadRatio) { multiValued = builder.getNumMultiValuesDocs() > 0; valueCount = builder.getValueCount(); PackedLongValues.Builder endOffsetsBuilder = PackedLongValues.monotonicBuilder(OFFSETS_PAGE_SIZE, acceptableOverheadRatio); PackedLongValues.Builder ordsBuilder = PackedLongValues.packedBuilder(OFFSETS_PAGE_SIZE, acceptableOverheadRatio); long lastEndOffset = 0; for (int i = 0; i < builder.maxDoc(); ++i) { final LongsRef docOrds = builder.docOrds(i); final long endOffset = lastEndOffset + docOrds.length; endOffsetsBuilder.add(endOffset); for (int j = 0; j < docOrds.length; ++j) { ordsBuilder.add(docOrds.longs[docOrds.offset + j]); } lastEndOffset = endOffset; } endOffsets = endOffsetsBuilder.build(); ords = ordsBuilder.build(); assert endOffsets.size() == builder.maxDoc(); assert ords.size() == builder.getTotalNumOrds() : ords.size() + " != " + builder.getTotalNumOrds(); }
@Override public long ramBytesUsed() { return endOffsets.ramBytesUsed() + ords.ramBytesUsed(); }
public SortedSetDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) { this.fieldInfo = fieldInfo; this.iwBytesUsed = iwBytesUsed; hash = new BytesRefHash( new ByteBlockPool( new ByteBlockPool.DirectTrackingAllocator(iwBytesUsed)), BytesRefHash.DEFAULT_CAPACITY, new DirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY, iwBytesUsed)); pending = PackedLongValues.packedBuilder(PackedInts.COMPACT); pendingCounts = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); docsWithField = new DocsWithFieldSet(); bytesUsed = pending.ramBytesUsed() + pendingCounts.ramBytesUsed(); iwBytesUsed.addAndGet(bytesUsed); }
/** @see #monotonicBuilder(int, float) */ public static PackedLongValues.Builder monotonicBuilder(float acceptableOverheadRatio) { return monotonicBuilder(DEFAULT_PAGE_SIZE, acceptableOverheadRatio); }
public void mergeBuckets(long[] mergeMap) { List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry sourceEntry : entries) { PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build())); } entries = newEntries; // if there are buckets that have been collected in the current segment // we need to update the bucket ordinals there too if (buckets.size() > 0) { PackedLongValues currentBuckets = buckets.build(); PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } buckets = newBuckets; } }
final PackedLongValues.Builder newToOldBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); for (int i = 0; i < maxDoc; ++i) { newToOldBuilder.add(docs[i]); docs[(int) newToOld.get(i)] = i; final PackedLongValues.Builder oldToNewBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); for (int i = 0; i < maxDoc; ++i) { oldToNewBuilder.add(docs[i]);
/** @see #deltaPackedBuilder(int, float) */ public static PackedLongValues.Builder deltaPackedBuilder(float acceptableOverheadRatio) { return deltaPackedBuilder(DEFAULT_PAGE_SIZE, acceptableOverheadRatio); }
private void fillBlock() { if (vOff == values.length) { currentCount = 0; } else { currentCount = decodeBlock(vOff, currentValues); assert currentCount > 0; } }
/** * Returns the total number of unique terms in global ord space. */ public long getValueCount() { return globalOrdDeltas.size(); }
/** Build a {@link PackedLongValues} instance that contains values that * have been added to this builder. This operation is destructive. */ public PackedLongValues build() { finish(); pending = null; final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff); final long ramBytesUsed = PackedLongValues.BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); return new PackedLongValues(pageShift, pageMask, values, size, ramBytesUsed); }
PackedLongValues.Builder globalOrdDeltas = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); PackedLongValues.Builder firstSegments = PackedLongValues.packedBuilder(PackedInts.COMPACT); final PackedLongValues.Builder[] ordDeltas = new PackedLongValues.Builder[subs.length]; for (int i = 0; i < ordDeltas.length; i++) { ordDeltas[i] = PackedLongValues.monotonicBuilder(acceptableOverheadRatio); long ramBytesUsed = BASE_RAM_BYTES_USED + this.globalOrdDeltas.ramBytesUsed() + this.firstSegments.ramBytesUsed() + RamUsageEstimator.shallowSizeOf(segmentToGlobalOrds) + segmentMap.ramBytesUsed(); for (int i = 0; i < ordDeltas.length; ++i) { } else { final int bitsRequired = ordDeltaBits[i] < 0 ? 64 : PackedInts.bitsRequired(ordDeltaBits[i]); final long monotonicBits = deltas.ramBytesUsed() * 8; final long packedBits = bitsRequired * deltas.size(); if (deltas.size() <= Integer.MAX_VALUE && packedBits <= monotonicBits * (1 + acceptableOverheadRatio)) { final int size = (int) deltas.size(); final PackedInts.Mutable newDeltas = PackedInts.getMutable(size, bitsRequired, acceptableOverheadRatio); final PackedLongValues.Iterator it = deltas.iterator(); for (int ord = 0; ord < size; ++ord) { newDeltas.set(ord, it.next()); ramBytesUsed += deltas.ramBytesUsed();
final LeafBucketCollector leafCollector = collector.getLeafCollector(entry.context); DocIdSetIterator docIt = null; if (needsScores && entry.docDeltas.size() > 0) { Scorer scorer = weight.scorer(entry.context); leafCollector.setScorer(scorer); final PackedLongValues.Iterator docDeltaIterator = entry.docDeltas.iterator(); final PackedLongValues.Iterator buckets = entry.buckets.iterator(); int doc = 0; for (long i = 0, end = entry.docDeltas.size(); i < end; ++i) { doc += docDeltaIterator.next(); final long bucket = buckets.next();
public MultiOrdinals(OrdinalsBuilder builder, float acceptableOverheadRatio) { multiValued = builder.getNumMultiValuesDocs() > 0; valueCount = builder.getValueCount(); PackedLongValues.Builder endOffsetsBuilder = PackedLongValues.monotonicBuilder(OFFSETS_PAGE_SIZE, acceptableOverheadRatio); PackedLongValues.Builder ordsBuilder = PackedLongValues.packedBuilder(OFFSETS_PAGE_SIZE, acceptableOverheadRatio); long lastEndOffset = 0; for (int i = 0; i < builder.maxDoc(); ++i) { final LongsRef docOrds = builder.docOrds(i); final long endOffset = lastEndOffset + docOrds.length; endOffsetsBuilder.add(endOffset); for (int j = 0; j < docOrds.length; ++j) { ordsBuilder.add(docOrds.longs[docOrds.offset + j]); } lastEndOffset = endOffset; } endOffsets = endOffsetsBuilder.build(); ords = ordsBuilder.build(); assert endOffsets.size() == builder.maxDoc(); assert ords.size() == builder.getTotalNumOrds() : ords.size() + " != " + builder.getTotalNumOrds(); }
@Override public long ramBytesUsed() { long size = ordinals.ramBytesUsed(); // PackedBytes size += bytes.ramBytesUsed(); // PackedInts size += termOrdToBytesOffset.ramBytesUsed(); return size; }