final long pointer = in.getFilePointer(); subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue)); } else { subReaders[i] = PackedInts.getReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
} else { for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { if (Format.PACKED_SINGLE_BLOCK.isSupported(bpv)) { float overhead = Format.PACKED_SINGLE_BLOCK.overheadPerValue(bpv); float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; if (overhead <= acceptableOverhead) {
final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), packedIntsVersion, entry.count, bitsPerValue); return new Norms() { @Override final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), packedIntsVersion, entry.count, bitsPerValue); final Norms nestedInstance = loadNorms(entry.nested);
data.writeVInt(tableBPV.format.getId()); data.writeVInt(tableBPV.bitsPerValue); data.writeLong(minValue); data.writeLong(gcd); data.writeVInt(gcdBPV.format.getId()); data.writeVInt(gcdBPV.bitsPerValue); final long minDelta = deltaBPV.bitsPerValue == 64 ? 0 : minValue; data.writeLong(minDelta); data.writeVInt(deltaBPV.format.getId()); data.writeVInt(deltaBPV.bitsPerValue);
final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, (int)entry.count, bitsPerValue); if (!merging) { ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed()); final int formatIDDelta = data.readVInt(); final int bitsPerValueDelta = data.readVInt(); final PackedInts.Reader deltaReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatIDDelta), entry.packedIntsVersion, (int)entry.count, bitsPerValueDelta); if (!merging) { ramBytesUsed.addAndGet(deltaReader.ramBytesUsed()); final int formatIDGCD = data.readVInt(); final int bitsPerValueGCD = data.readVInt(); final PackedInts.Reader quotientReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatIDGCD), entry.packedIntsVersion, (int)entry.count, bitsPerValueGCD); if (!merging) { ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); if (!merging) { ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), packedVersion, maxDoc, bitsPerValue); if (!merging) { ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
final long singleValuesSize = formatAndBits.format.longCount(PackedInts.VERSION_CURRENT, reader.maxDoc(), formatAndBits.bitsPerValue) * 8L;
} else { for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { if (Format.PACKED_SINGLE_BLOCK.isSupported(bpv)) { float overhead = Format.PACKED_SINGLE_BLOCK.overheadPerValue(bpv); float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; if (overhead <= acceptableOverhead) {
} else { for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { if (Format.PACKED_SINGLE_BLOCK.isSupported(bpv)) { float overhead = Format.PACKED_SINGLE_BLOCK.overheadPerValue(bpv); float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; if (overhead <= acceptableOverhead) {
} else { for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { if (Format.PACKED_SINGLE_BLOCK.isSupported(bpv)) { float overhead = Format.PACKED_SINGLE_BLOCK.overheadPerValue(bpv); float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; if (overhead <= acceptableOverhead) {
super(valueCount, bitsPerValue); final PackedInts.Format format = PackedInts.Format.PACKED; final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array blocks = new long[longCount];
super(valueCount, bitsPerValue); final PackedInts.Format format = PackedInts.Format.PACKED; final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array blocks = new long[longCount];
super(valueCount, bitsPerValue); final PackedInts.Format format = PackedInts.Format.PACKED; final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array blocks = new long[longCount];
/** * Restore a {@link ForUtil} from a {@link DataInput}. */ ForUtil(DataInput in) throws IOException { int packedIntsVersion = in.readVInt(); PackedInts.checkVersion(packedIntsVersion); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final int code = in.readVInt(); final int formatId = code >>> 5; final int bitsPerValue = (code & 31) + 1; final PackedInts.Format format = PackedInts.Format.byId(formatId); assert format.isSupported(bitsPerValue); encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue); encoders[bpv] = PackedInts.getEncoder( format, packedIntsVersion, bitsPerValue); decoders[bpv] = PackedInts.getDecoder( format, packedIntsVersion, bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); } }
/** * Restore a {@link ForUtil} from a {@link DataInput}. */ ForUtil(DataInput in) throws IOException { int packedIntsVersion = in.readVInt(); PackedInts.checkVersion(packedIntsVersion); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final int code = in.readVInt(); final int formatId = code >>> 5; final int bitsPerValue = (code & 31) + 1; final PackedInts.Format format = PackedInts.Format.byId(formatId); assert format.isSupported(bitsPerValue); encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue); encoders[bpv] = PackedInts.getEncoder( format, packedIntsVersion, bitsPerValue); decoders[bpv] = PackedInts.getDecoder( format, packedIntsVersion, bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); } }
private long getPageMemoryUsage(PackedLongValues values, float acceptableOverheadRatio, int pageSize, long pageMinOrdinal, long pageMaxOrdinal) { int bitsRequired; long pageMemorySize = 0; PackedInts.FormatAndBits formatAndBits; if (pageMaxOrdinal == Long.MIN_VALUE) { // empty page - will use the null reader which just stores size pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT); } else { long pageMinValue = values.get(pageMinOrdinal); long pageMaxValue = values.get(pageMaxOrdinal); long pageDelta = pageMaxValue - pageMinValue; if (pageDelta != 0) { bitsRequired = pageDelta < 0 ? 64 : PackedInts.bitsRequired(pageDelta); formatAndBits = PackedInts.fastestFormatAndBits(pageSize, bitsRequired, acceptableOverheadRatio); pageMemorySize += formatAndBits.format.longCount(PackedInts.VERSION_CURRENT, pageSize, formatAndBits.bitsPerValue) * RamUsageEstimator.NUM_BYTES_LONG; pageMemorySize += RamUsageEstimator.NUM_BYTES_LONG; // min value per page storage } else { // empty page pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT); } } return pageMemorySize; }
/** * Create a new {@link ForUtil} instance and save state into <code>out</code>. */ ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( BLOCK_SIZE, bpv, acceptableOverheadRatio); assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); assert formatAndBits.bitsPerValue <= 32; encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); encoders[bpv] = PackedInts.getEncoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); decoders[bpv] = PackedInts.getDecoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); } }
/** * Restore a {@link ForUtil} from a {@link DataInput}. */ ForUtil(DataInput in) throws IOException { int packedIntsVersion = in.readVInt(); PackedInts.checkVersion(packedIntsVersion); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final int code = in.readVInt(); final int formatId = code >>> 5; final int bitsPerValue = (code & 31) + 1; final PackedInts.Format format = PackedInts.Format.byId(formatId); assert format.isSupported(bitsPerValue); encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue); encoders[bpv] = PackedInts.getEncoder( format, packedIntsVersion, bitsPerValue); decoders[bpv] = PackedInts.getDecoder( format, packedIntsVersion, bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); } }