/** * Create a new {@link PagedMutable} instance. * * @param size the number of values to store. * @param pageSize the number of values per page * @param bitsPerValue the number of bits per value * @param acceptableOverheadRatio an acceptable overhead ratio */ public PagedMutable(long size, int pageSize, int bitsPerValue, float acceptableOverheadRatio) { this(size, pageSize, PackedInts.fastestFormatAndBits(pageSize, bitsPerValue, acceptableOverheadRatio)); fillPages(); }
/** * Create a packed integer array with the given amount of values initialized * to 0. the valueCount and the bitsPerValue cannot be changed after creation. * All Mutables known by this factory are kept fully in RAM. * <p> * Positive values of <code>acceptableOverheadRatio</code> will trade space * for speed by selecting a faster but potentially less memory-efficient * implementation. An <code>acceptableOverheadRatio</code> of * {@link PackedInts#COMPACT} will make sure that the most memory-efficient * implementation is selected whereas {@link PackedInts#FASTEST} will make sure * that the fastest implementation is selected. * * @param valueCount the number of elements * @param bitsPerValue the number of bits available for any given value * @param acceptableOverheadRatio an acceptable overhead * ratio per value * @return a mutable packed integer array * @lucene.internal */ public static Mutable getMutable(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); return getMutable(valueCount, formatAndBits.bitsPerValue, formatAndBits.format); }
assert valueCount >= 0; final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); final Writer writer = getWriterNoHeader(out, formatAndBits.format, valueCount, formatAndBits.bitsPerValue, DEFAULT_BUFFER_SIZE); writer.writeHeader();
/** * Create a new {@link ForUtil} instance and save state into <code>out</code>. */ ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( BLOCK_SIZE, bpv, acceptableOverheadRatio); assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); assert formatAndBits.bitsPerValue <= 32; encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); encoders[bpv] = PackedInts.getEncoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); decoders[bpv] = PackedInts.getDecoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); } }
/** * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. */ public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds, float acceptableOverheadRatio) { int bitsPerOrd = PackedInts.bitsRequired(numOrds); bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue; // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the // beginning of the block and all docs have one at the end of the block final float avgValuesPerDoc = (float) numDocsWithValue / maxDoc; final int maxDelta = (int) Math.ceil(OFFSETS_PAGE_SIZE * (1 - avgValuesPerDoc) * avgValuesPerDoc); int bitsPerOffset = PackedInts.bitsRequired(maxDelta) + 1; // +1 because of the sign bitsPerOffset = PackedInts.fastestFormatAndBits(maxDoc, bitsPerOffset, acceptableOverheadRatio).bitsPerValue; final long expectedMultiSizeInBytes = (long) numDocsWithValue * bitsPerOrd + (long) maxDoc * bitsPerOffset; final long expectedSingleSizeInBytes = (long) maxDoc * bitsPerOrd; return expectedMultiSizeInBytes < 0.8f * expectedSingleSizeInBytes; }
/** * Create a new {@link PagedMutable} instance. * * @param size the number of values to store. * @param pageSize the number of values per page * @param bitsPerValue the number of bits per value * @param acceptableOverheadRatio an acceptable overhead ratio */ public PagedMutable(long size, int pageSize, int bitsPerValue, float acceptableOverheadRatio) { this(size, pageSize, PackedInts.fastestFormatAndBits(pageSize, bitsPerValue, acceptableOverheadRatio)); fillPages(); }
/** * Create a new {@link PagedMutable} instance. * * @param size the number of values to store. * @param pageSize the number of values per page * @param bitsPerValue the number of bits per value * @param acceptableOverheadRatio an acceptable overhead ratio */ public PagedMutable(long size, int pageSize, int bitsPerValue, float acceptableOverheadRatio) { this(size, pageSize, PackedInts.fastestFormatAndBits(pageSize, bitsPerValue, acceptableOverheadRatio)); fillPages(); }
/** * Create a new {@link PagedMutable} instance. * * @param size the number of values to store. * @param pageSize the number of values per page * @param bitsPerValue the number of bits per value * @param acceptableOverheadRatio an acceptable overhead ratio */ public PagedMutable(long size, int pageSize, int bitsPerValue, float acceptableOverheadRatio) { this(size, pageSize, PackedInts.fastestFormatAndBits(pageSize, bitsPerValue, acceptableOverheadRatio)); fillPages(); }
/** * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. */ public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds, float acceptableOverheadRatio) { int bitsPerOrd = PackedInts.bitsRequired(numOrds); bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue; // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the // beginning of the block and all docs have one at the end of the block final float avgValuesPerDoc = (float) numDocsWithValue / maxDoc; final int maxDelta = (int) Math.ceil(OFFSETS_PAGE_SIZE * (1 - avgValuesPerDoc) * avgValuesPerDoc); int bitsPerOffset = PackedInts.bitsRequired(maxDelta) + 1; // +1 because of the sign bitsPerOffset = PackedInts.fastestFormatAndBits(maxDoc, bitsPerOffset, acceptableOverheadRatio).bitsPerValue; final long expectedMultiSizeInBytes = (long) numDocsWithValue * bitsPerOrd + (long) maxDoc * bitsPerOffset; final long expectedSingleSizeInBytes = (long) maxDoc * bitsPerOrd; return expectedMultiSizeInBytes < 0.8f * expectedSingleSizeInBytes; }
/** * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. */ public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds, float acceptableOverheadRatio) { int bitsPerOrd = PackedInts.bitsRequired(numOrds); bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue; // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the // beginning of the block and all docs have one at the end of the block final float avgValuesPerDoc = (float) numDocsWithValue / maxDoc; final int maxDelta = (int) Math.ceil(OFFSETS_PAGE_SIZE * (1 - avgValuesPerDoc) * avgValuesPerDoc); int bitsPerOffset = PackedInts.bitsRequired(maxDelta) + 1; // +1 because of the sign bitsPerOffset = PackedInts.fastestFormatAndBits(maxDoc, bitsPerOffset, acceptableOverheadRatio).bitsPerValue; final long expectedMultiSizeInBytes = (long) numDocsWithValue * bitsPerOrd + (long) maxDoc * bitsPerOffset; final long expectedSingleSizeInBytes = (long) maxDoc * bitsPerOrd; return expectedMultiSizeInBytes < 0.8f * expectedSingleSizeInBytes; }
/** * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. */ public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds, float acceptableOverheadRatio) { int bitsPerOrd = PackedInts.bitsRequired(numOrds); bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue; // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the // beginning of the block and all docs have one at the end of the block final float avgValuesPerDoc = (float) numDocsWithValue / maxDoc; final int maxDelta = (int) Math.ceil(OFFSETS_PAGE_SIZE * (1 - avgValuesPerDoc) * avgValuesPerDoc); int bitsPerOffset = PackedInts.bitsRequired(maxDelta) + 1; // +1 because of the sign bitsPerOffset = PackedInts.fastestFormatAndBits(maxDoc, bitsPerOffset, acceptableOverheadRatio).bitsPerValue; final long expectedMultiSizeInBytes = (long) numDocsWithValue * bitsPerOrd + (long) maxDoc * bitsPerOffset; final long expectedSingleSizeInBytes = (long) maxDoc * bitsPerOrd; return expectedMultiSizeInBytes < 0.8f * expectedSingleSizeInBytes; }
/** * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. */ public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds, float acceptableOverheadRatio) { int bitsPerOrd = PackedInts.bitsRequired(numOrds); bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue; // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the // beginning of the block and all docs have one at the end of the block final float avgValuesPerDoc = (float) numDocsWithValue / maxDoc; final int maxDelta = (int) Math.ceil(OFFSETS_PAGE_SIZE * (1 - avgValuesPerDoc) * avgValuesPerDoc); int bitsPerOffset = PackedInts.bitsRequired(maxDelta) + 1; // +1 because of the sign bitsPerOffset = PackedInts.fastestFormatAndBits(maxDoc, bitsPerOffset, acceptableOverheadRatio).bitsPerValue; final long expectedMultiSizeInBytes = (long) numDocsWithValue * bitsPerOrd + (long) maxDoc * bitsPerOffset; final long expectedSingleSizeInBytes = (long) maxDoc * bitsPerOrd; return expectedMultiSizeInBytes < 0.8f * expectedSingleSizeInBytes; }
/** * Create a packed integer array with the given amount of values initialized * to 0. the valueCount and the bitsPerValue cannot be changed after creation. * All Mutables known by this factory are kept fully in RAM. * <p> * Positive values of <code>acceptableOverheadRatio</code> will trade space * for speed by selecting a faster but potentially less memory-efficient * implementation. An <code>acceptableOverheadRatio</code> of * {@link PackedInts#COMPACT} will make sure that the most memory-efficient * implementation is selected whereas {@link PackedInts#FASTEST} will make sure * that the fastest implementation is selected. * * @param valueCount the number of elements * @param bitsPerValue the number of bits available for any given value * @param acceptableOverheadRatio an acceptable overhead * ratio per value * @return a mutable packed integer array * @lucene.internal */ public static Mutable getMutable(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); return getMutable(valueCount, formatAndBits.bitsPerValue, formatAndBits.format); }
/** * Create a packed integer array with the given amount of values initialized * to 0. the valueCount and the bitsPerValue cannot be changed after creation. * All Mutables known by this factory are kept fully in RAM. * <p> * Positive values of <code>acceptableOverheadRatio</code> will trade space * for speed by selecting a faster but potentially less memory-efficient * implementation. An <code>acceptableOverheadRatio</code> of * {@link PackedInts#COMPACT} will make sure that the most memory-efficient * implementation is selected whereas {@link PackedInts#FASTEST} will make sure * that the fastest implementation is selected. * * @param valueCount the number of elements * @param bitsPerValue the number of bits available for any given value * @param acceptableOverheadRatio an acceptable overhead * ratio per value * @return a mutable packed integer array * @lucene.internal */ public static Mutable getMutable(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); return getMutable(valueCount, formatAndBits.bitsPerValue, formatAndBits.format); }
/** * Create a packed integer array with the given amount of values initialized * to 0. the valueCount and the bitsPerValue cannot be changed after creation. * All Mutables known by this factory are kept fully in RAM. * <p> * Positive values of <code>acceptableOverheadRatio</code> will trade space * for speed by selecting a faster but potentially less memory-efficient * implementation. An <code>acceptableOverheadRatio</code> of * {@link PackedInts#COMPACT} will make sure that the most memory-efficient * implementation is selected whereas {@link PackedInts#FASTEST} will make sure * that the fastest implementation is selected. * * @param valueCount the number of elements * @param bitsPerValue the number of bits available for any given value * @param acceptableOverheadRatio an acceptable overhead * ratio per value * @return a mutable packed integer array * @lucene.internal */ public static Mutable getMutable(int valueCount, int bitsPerValue, float acceptableOverheadRatio) { final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio); return getMutable(valueCount, formatAndBits.bitsPerValue, formatAndBits.format); }
private long getPageMemoryUsage(PackedLongValues values, float acceptableOverheadRatio, int pageSize, long pageMinOrdinal, long pageMaxOrdinal) { int bitsRequired; long pageMemorySize = 0; PackedInts.FormatAndBits formatAndBits; if (pageMaxOrdinal == Long.MIN_VALUE) { // empty page - will use the null reader which just stores size pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT); } else { long pageMinValue = values.get(pageMinOrdinal); long pageMaxValue = values.get(pageMaxOrdinal); long pageDelta = pageMaxValue - pageMinValue; if (pageDelta != 0) { bitsRequired = pageDelta < 0 ? 64 : PackedInts.bitsRequired(pageDelta); formatAndBits = PackedInts.fastestFormatAndBits(pageSize, bitsRequired, acceptableOverheadRatio); pageMemorySize += formatAndBits.format.longCount(PackedInts.VERSION_CURRENT, pageSize, formatAndBits.bitsPerValue) * RamUsageEstimator.NUM_BYTES_LONG; pageMemorySize += RamUsageEstimator.NUM_BYTES_LONG; // min value per page storage } else { // empty page pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT); } } return pageMemorySize; }
/** * Create a new {@link ForUtil} instance and save state into <code>out</code>. */ ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( BLOCK_SIZE, bpv, acceptableOverheadRatio); assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); assert formatAndBits.bitsPerValue <= 32; encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); encoders[bpv] = PackedInts.getEncoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); decoders[bpv] = PackedInts.getDecoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); } }
/** * Create a new {@link ForUtil} instance and save state into <code>out</code>. */ ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( BLOCK_SIZE, bpv, acceptableOverheadRatio); assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); assert formatAndBits.bitsPerValue <= 32; encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); encoders[bpv] = PackedInts.getEncoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); decoders[bpv] = PackedInts.getDecoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); } }
/** * Create a new {@link ForUtil} instance and save state into <code>out</code>. */ ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( BLOCK_SIZE, bpv, acceptableOverheadRatio); assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); assert formatAndBits.bitsPerValue <= 32; encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); encoders[bpv] = PackedInts.getEncoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); decoders[bpv] = PackedInts.getDecoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); } }
/** * Create a new {@link ForUtil} instance and save state into <code>out</code>. */ ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( BLOCK_SIZE, bpv, acceptableOverheadRatio); assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); assert formatAndBits.bitsPerValue <= 32; encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); encoders[bpv] = PackedInts.getEncoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); decoders[bpv] = PackedInts.getDecoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); } }