private void flushFields(int totalFields, int[] fieldNums) throws IOException { final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, PackedInts.bitsRequired(fieldNums.length - 1), 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum); assert fieldNumIndex >= 0; writer.add(fieldNumIndex); } } writer.finish(); }
/** * Create a new {@link ForUtil} instance and save state into <code>out</code>. */ ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( BLOCK_SIZE, bpv, acceptableOverheadRatio); assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); assert formatAndBits.bitsPerValue <= 32; encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); encoders[bpv] = PackedInts.getEncoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); decoders[bpv] = PackedInts.getDecoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); } }
private void ensureCapacity(long value) { if ((value & currentMask) == value) { return; } final int bitsRequired = PackedInts.unsignedBitsRequired(value); assert bitsRequired > current.getBitsPerValue(); final int valueCount = size(); PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio); PackedInts.copy(current, 0, next, 0, valueCount, PackedInts.DEFAULT_BUFFER_SIZE); current = next; currentMask = mask(current.getBitsPerValue()); }
AbstractPagedMutable(int bitsPerValue, long size, int pageSize) { this.bitsPerValue = bitsPerValue; this.size = size; pageShift = checkBlockSize(pageSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE); pageMask = pageSize - 1; final int numPages = numBlocks(size, pageSize); subMutables = new PackedInts.Mutable[numPages]; }
/** * Create a packed integer array with the given amount of values initialized * to 0. the valueCount and the bitsPerValue cannot be changed after creation. * All Mutables known by this factory are kept fully in RAM. * <p> * Positive values of <code>acceptableOverheadRatio</code> will trade space * for speed by selecting a faster but potentially less memory-efficient * implementation. An <code>acceptableOverheadRatio</code> of * {@link PackedInts#COMPACT} will make sure that the most memory-efficient * implementation is selected whereas {@link PackedInts#FASTEST} will make sure * that the fastest implementation is selected. * * @param valueCount the number of elements * @param bitsPerValue the number of bits available for any given value * @param acceptableOverheadRatio an acceptable overhead * ratio per value * @return a mutable packed integer array * @lucene.internal */ public static Mutable getMutable(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); return getMutable(valueCount, formatAndBits.bitsPerValue, formatAndBits.format); }
private MonotonicBlockPackedReader(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException { this.valueCount = valueCount; blockShift = checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE); blockMask = blockSize - 1; final int numBlocks = numBlocks(valueCount, blockSize); minValues = new long[numBlocks]; averages = new float[numBlocks]; if (direct) { final long pointer = in.getFilePointer(); subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue)); } else { subReaders[i] = PackedInts.getReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
public SinglePackedOrdinals(OrdinalsBuilder builder, float acceptableOverheadRatio) { assert builder.getNumMultiValuesDocs() == 0; this.valueCount = (int) builder.getValueCount(); // We don't reuse the builder as-is because it might have been built with a higher overhead ratio final PackedInts.Mutable reader = PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(valueCount), acceptableOverheadRatio); PackedInts.copy(builder.getFirstOrdinals(), 0, reader, 0, builder.maxDoc(), 8 * 1024); this.reader = reader; }
/** * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. */ public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds, float acceptableOverheadRatio) { int bitsPerOrd = PackedInts.bitsRequired(numOrds); bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue; // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the // beginning of the block and all docs have one at the end of the block final float avgValuesPerDoc = (float) numDocsWithValue / maxDoc; final int maxDelta = (int) Math.ceil(OFFSETS_PAGE_SIZE * (1 - avgValuesPerDoc) * avgValuesPerDoc); int bitsPerOffset = PackedInts.bitsRequired(maxDelta) + 1; // +1 because of the sign bitsPerOffset = PackedInts.fastestFormatAndBits(maxDoc, bitsPerOffset, acceptableOverheadRatio).bitsPerValue; final long expectedMultiSizeInBytes = (long) numDocsWithValue * bitsPerOrd + (long) maxDoc * bitsPerOffset; final long expectedSingleSizeInBytes = (long) maxDoc * bitsPerOrd; return expectedMultiSizeInBytes < 0.8f * expectedSingleSizeInBytes; }
void reset(int len) { final int bitsPerOffset = PackedInts.bitsRequired(len - LAST_LITERALS); final int bitsPerOffsetLog = 32 - Integer.numberOfLeadingZeros(bitsPerOffset - 1); hashLog = MEMORY_USAGE + 3 - bitsPerOffsetLog; if (hashTable == null || hashTable.size() < 1 << hashLog || hashTable.getBitsPerValue() < bitsPerOffset) { hashTable = PackedInts.getMutable(1 << hashLog, bitsPerOffset, PackedInts.DEFAULT); } else { hashTable.clear(); } }
/** * Restore a {@link ForUtil} from a {@link DataInput}. */ ForUtil(DataInput in) throws IOException { int packedIntsVersion = in.readVInt(); PackedInts.checkVersion(packedIntsVersion); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final int code = in.readVInt(); final int formatId = code >>> 5; final int bitsPerValue = (code & 31) + 1; final PackedInts.Format format = PackedInts.Format.byId(formatId); assert format.isSupported(bitsPerValue); encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue); encoders[bpv] = PackedInts.getEncoder( format, packedIntsVersion, bitsPerValue); decoders[bpv] = PackedInts.getDecoder( format, packedIntsVersion, bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); } }
final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalDistinctFields, bitsPerFieldNum, 1); fieldNums = new int[totalDistinctFields]; for (int i = 0; i < totalDistinctFields; ++i) { final PackedInts.Reader flags; final int bitsPerOff = PackedInts.bitsRequired(fieldNums.length - 1); final PackedInts.Reader allFieldNumOffs = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsPerOff); switch (vectorsStream.readVInt()) { case 0: final PackedInts.Reader fieldFlags = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, fieldNums.length, FLAGS_BITS); PackedInts.Mutable f = PackedInts.getMutable(totalFields, FLAGS_BITS, PackedInts.COMPACT); for (int i = 0; i < totalFields; ++i) { final int fieldNumOff = (int) allFieldNumOffs.get(i); break; case 1: flags = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, FLAGS_BITS); break; default: numTerms = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsRequired); int sum = 0; for (int i = 0; i < totalFields; ++i) {
/** * Compute the number of bits required to serialize any of the longs in * <code>data</code>. */ private static int bitsRequired(final int[] data) { long or = 0; for (int i = 0; i < BLOCK_SIZE; ++i) { assert data[i] >= 0; or |= data[i]; } return PackedInts.bitsRequired(or); }
assert valueCount >= 0; final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); final Writer writer = getWriterNoHeader(out, formatAndBits.format, valueCount, formatAndBits.bitsPerValue, DEFAULT_BUFFER_SIZE); writer.writeHeader(); return writer;
/** * Create a new {@link PagedMutable} instance. * * @param size the number of values to store. * @param pageSize the number of values per page * @param bitsPerValue the number of bits per value * @param acceptableOverheadRatio an acceptable overhead ratio */ public PagedMutable(long size, int pageSize, int bitsPerValue, float acceptableOverheadRatio) { this(size, pageSize, PackedInts.fastestFormatAndBits(pageSize, bitsPerValue, acceptableOverheadRatio)); fillPages(); }
public GrowableWriter resize(int newSize) { GrowableWriter next = new GrowableWriter(getBitsPerValue(), newSize, acceptableOverheadRatio); final int limit = Math.min(size(), newSize); PackedInts.copy(current, 0, next, 0, limit, PackedInts.DEFAULT_BUFFER_SIZE); return next; }
/** * Get a {@link Decoder}. * * @param format the format used to store packed ints * @param version the compatibility version * @param bitsPerValue the number of bits per value * @return a decoder */ public static Decoder getDecoder(Format format, int version, int bitsPerValue) { checkVersion(version); return BulkOperation.of(format, bitsPerValue); }
@Override public void encode(long[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations) { int nextBlock = 0; int bitsLeft = 8; for (int i = 0; i < byteValueCount * iterations; ++i) { final long v = values[valuesOffset++]; assert PackedInts.unsignedBitsRequired(v) <= bitsPerValue; if (bitsPerValue < bitsLeft) { // just buffer nextBlock |= v << (bitsLeft - bitsPerValue); bitsLeft -= bitsPerValue; } else { // flush as many blocks as possible int bits = bitsPerValue - bitsLeft; blocks[blocksOffset++] = (byte) (nextBlock | (v >>> bits)); while (bits >= 8) { bits -= 8; blocks[blocksOffset++] = (byte) (v >>> bits); } // then buffer bitsLeft = 8 - bits; nextBlock = (int) ((v & ((1L << bits) - 1)) << bitsLeft); } } assert bitsLeft == 8; }
/** * Restore a {@link Reader} from a stream. * * @param in the stream to read data from * @return a Reader * @throws IOException If there is a low-level I/O error * @lucene.internal */ public static Reader getReader(DataInput in) throws IOException { final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT); final int bitsPerValue = in.readVInt(); assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; final int valueCount = in.readVInt(); final Format format = Format.byId(in.readVInt()); return getReaderNoHeader(in, format, version, valueCount, bitsPerValue); }
/** * Sole constructor. * @param blockSize the number of values of a single block, must be a multiple of <tt>64</tt> */ public AbstractBlockPackedWriter(DataOutput out, int blockSize) { checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE); reset(out); values = new long[blockSize]; }
int bitsRequired = delta == 0 ? 0 : PackedInts.unsignedBitsRequired(delta); if (bitsRequired == 64) { } else if (min > 0L) { min = Math.max(0L, max - PackedInts.maxValue(bitsRequired));