meta.writeByte((byte) 0); } else { final int bitsRequired = DirectWriter.unsignedBitsRequired(maxDelta); DirectWriter writer = DirectWriter.getInstance(data, bufferSize, bitsRequired); for (int i = 0; i < bufferSize; ++i) { writer.add(buffer[i]); writer.finish(); meta.writeByte((byte) bitsRequired);
/** Returns an instance suitable for encoding {@code numValues} using {@code bitsPerValue} */ public static DirectWriter getInstance(DataOutput output, long numValues, int bitsPerValue) { if (Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) < 0) { throw new IllegalArgumentException("Unsupported bitsPerValue " + bitsPerValue + ". Did you use bitsRequired?"); } return new DirectWriter(output, numValues, bitsPerValue); }
/** Adds a value to this writer */ public void add(long l) throws IOException { assert bitsPerValue == 64 || (l >= 0 && l <= PackedInts.maxValue(bitsPerValue)) : bitsPerValue; assert !finished; if (count >= numValues) { throw new EOFException("Writing past end of stream"); } nextValues[off++] = l; if (off == nextValues.length) { flush(); } count++; }
private void writeValuesSingleBlock(SortedNumericDocValues values, long numValues, int numBitsPerValue, long min, long gcd, Map<Long, Integer> encode) throws IOException { DirectWriter writer = DirectWriter.getInstance(data, numValues, numBitsPerValue); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { for (int i = 0, count = values.docValueCount(); i < count; ++i) { long v = values.nextValue(); if (encode == null) { writer.add((v - min) / gcd); } else { writer.add(encode.get(v)); } } } writer.finish(); }
/** * Returns how many bits are required to hold values up * to and including maxValue * * @param maxValue the maximum value that should be representable. * @return the amount of bits needed to represent values from 0 to maxValue. * @see PackedInts#bitsRequired(long) */ public static int bitsRequired(long maxValue) { return roundBits(PackedInts.bitsRequired(maxValue)); }
/** Update the required space. */ void finish() { if (max > min) { spaceInBits += DirectWriter.unsignedBitsRequired(max - min) * numValues; } }
private void writeValuesSingleBlock(SortedNumericDocValues values, long numValues, int numBitsPerValue, long min, long gcd, Map<Long, Integer> encode) throws IOException { DirectWriter writer = DirectWriter.getInstance(data, numValues, numBitsPerValue); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { for (int i = 0, count = values.docValueCount(); i < count; ++i) { long v = values.nextValue(); if (encode == null) { writer.add((v - min) / gcd); } else { writer.add(encode.get(v)); } } } writer.finish(); }
/** * Returns how many bits are required to hold values up * to and including maxValue, interpreted as an unsigned value. * * @param maxValue the maximum value that should be representable. * @return the amount of bits needed to represent values from 0 to maxValue. * @see PackedInts#unsignedBitsRequired(long) */ public static int unsignedBitsRequired(long maxValue) { return roundBits(PackedInts.unsignedBitsRequired(maxValue)); }
if (uniqueValues != null && uniqueValues.size() > 1 && DirectWriter.unsignedBitsRequired(uniqueValues.size() - 1) < DirectWriter.unsignedBitsRequired((max - min) / gcd)) { numBitsPerValue = DirectWriter.unsignedBitsRequired(uniqueValues.size() - 1); final Long[] sortedUniqueValues = uniqueValues.toArray(new Long[0]); Arrays.sort(sortedUniqueValues); meta.writeInt(-2 - NUMERIC_BLOCK_SHIFT); } else { numBitsPerValue = DirectWriter.unsignedBitsRequired((max - min) / gcd); if (gcd == 1 && min > 0 && DirectWriter.unsignedBitsRequired(max) == DirectWriter.unsignedBitsRequired(max - min)) { min = 0;
private void writeBlock(long[] values, int length, long gcd, GrowableByteArrayDataOutput buffer) throws IOException { assert length > 0; long min = values[0]; long max = values[0]; for (int i = 1; i < length; ++i) { final long v = values[i]; assert Math.floorMod(values[i] - min, gcd) == 0; min = Math.min(min, v); max = Math.max(max, v); } if (min == max) { data.writeByte((byte) 0); data.writeLong(min); } else { final int bitsPerValue = DirectWriter.unsignedBitsRequired(max - min); buffer.reset(); assert buffer.getPosition() == 0; final DirectWriter w = DirectWriter.getInstance(buffer, length, bitsPerValue); for (int i = 0; i < length; ++i) { w.add((values[i] - min) / gcd); } w.finish(); data.writeByte((byte) bitsPerValue); data.writeLong(min); data.writeInt(buffer.getPosition()); data.writeBytes(buffer.getBytes(), buffer.getPosition()); } }
/** * Returns how many bits are required to hold values up * to and including maxValue, interpreted as an unsigned value. * * @param maxValue the maximum value that should be representable. * @return the amount of bits needed to represent values from 0 to maxValue. * @see PackedInts#unsignedBitsRequired(long) */ public static int unsignedBitsRequired(long maxValue) { return roundBits(PackedInts.unsignedBitsRequired(maxValue)); }
/** finishes writing */ public void finish() throws IOException { if (count != numValues) { throw new IllegalStateException("Wrong number of values added, expected: " + numValues + ", got: " + count); } assert !finished; flush(); // pad for fast io: we actually only need this for certain BPV, but its just 3 bytes... for (int i = 0; i < 3; i++) { output.writeByte((byte) 0); } finished = true; }
/** Update the required space. */ void finish() { if (max > min) { spaceInBits += DirectWriter.unsignedBitsRequired(max - min) * numValues; } }
/** Returns an instance suitable for encoding {@code numValues} using {@code bitsPerValue} */ public static DirectWriter getInstance(DataOutput output, long numValues, int bitsPerValue) { if (Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) < 0) { throw new IllegalArgumentException("Unsupported bitsPerValue " + bitsPerValue + ". Did you use bitsRequired?"); } return new DirectWriter(output, numValues, bitsPerValue); }
meta.writeLong(0L); } else { int numberOfBitsPerOrd = DirectWriter.unsignedBitsRequired(values.getValueCount() - 1); meta.writeByte((byte) numberOfBitsPerOrd); long start = data.getFilePointer(); meta.writeLong(start); DirectWriter writer = DirectWriter.getInstance(data, numDocsWithField, numberOfBitsPerOrd); values = valuesProducer.getSorted(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { writer.add(values.ordValue()); writer.finish(); meta.writeLong(data.getFilePointer() - start);
/** * Returns how many bits are required to hold values up * to and including maxValue * * @param maxValue the maximum value that should be representable. * @return the amount of bits needed to represent values from 0 to maxValue. * @see PackedInts#bitsRequired(long) */ public static int bitsRequired(long maxValue) { return roundBits(PackedInts.bitsRequired(maxValue)); }
/** Adds a value to this writer */ public void add(long l) throws IOException { assert bitsPerValue == 64 || (l >= 0 && l <= PackedInts.maxValue(bitsPerValue)) : bitsPerValue; assert !finished; if (count >= numValues) { throw new EOFException("Writing past end of stream"); } nextValues[off++] = l; if (off == nextValues.length) { flush(); } count++; }
if (uniqueValues != null && uniqueValues.size() > 1 && DirectWriter.unsignedBitsRequired(uniqueValues.size() - 1) < DirectWriter.unsignedBitsRequired((max - min) / gcd)) { numBitsPerValue = DirectWriter.unsignedBitsRequired(uniqueValues.size() - 1); final Long[] sortedUniqueValues = uniqueValues.toArray(new Long[0]); Arrays.sort(sortedUniqueValues); meta.writeInt(-2 - NUMERIC_BLOCK_SHIFT); } else { numBitsPerValue = DirectWriter.unsignedBitsRequired((max - min) / gcd); if (gcd == 1 && min > 0 && DirectWriter.unsignedBitsRequired(max) == DirectWriter.unsignedBitsRequired(max - min)) { min = 0;
/** Returns an instance suitable for encoding {@code numValues} using {@code bitsPerValue} */ public static DirectWriter getInstance(IndexOutput output, long numValues, int bitsPerValue) { if (Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) < 0) { throw new IllegalArgumentException("Unsupported bitsPerValue " + bitsPerValue + ". Did you use bitsRequired?"); } return new DirectWriter(output, numValues, bitsPerValue); }
int numberOfBitsPerOrd = DirectWriter.unsignedBitsRequired(values.getValueCount() - 1); meta.writeByte((byte) numberOfBitsPerOrd); long start = data.getFilePointer(); meta.writeLong(start); DirectWriter writer = DirectWriter.getInstance(data, numOrds, numberOfBitsPerOrd); values = valuesProducer.getSortedSet(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { writer.add(ord); writer.finish(); meta.writeLong(data.getFilePointer() - start);