private void ensureCapacity(long value) { if ((value & currentMask) == value) { return; } final int bitsRequired = PackedInts.unsignedBitsRequired(value); assert bitsRequired > current.getBitsPerValue(); final int valueCount = size(); PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio); PackedInts.copy(current, 0, next, 0, valueCount, PackedInts.DEFAULT_BUFFER_SIZE); current = next; currentMask = mask(current.getBitsPerValue()); }
@Override public final long get(long index) { assert index >= 0 && index < size: "index=" + index + " size=" + size; final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); return subMutables[pageIndex].get(indexInPage); }
private void pack() { pack(pending, pendingOff, valuesOff, acceptableOverheadRatio); ramBytesUsed += values[valuesOff].ramBytesUsed(); valuesOff += 1; // reset pending buffer pendingOff = 0; }
/** * Save this mutable into <code>out</code>. Instantiating a reader from * the generated data will return a reader with the same number of bits * per value. */ public void save(DataOutput out) throws IOException { Writer writer = getWriterNoHeader(out, getFormat(), size(), getBitsPerValue(), DEFAULT_BUFFER_SIZE); writer.writeHeader(); for (int i = 0; i < size(); ++i) { writer.add(get(i)); } writer.finish(); }
protected final void fillPages() { final int numPages = numBlocks(size, pageSize()); for (int i = 0; i < numPages; ++i) { // do not allocate for more entries than necessary on the last page final int valueCount = i == numPages - 1 ? lastPageSize(size) : pageSize(); subMutables[i] = newMutable(valueCount, bitsPerValue); } }
/** * @param startBitsPerValue the initial number of bits per value, may grow depending on the data * @param valueCount the number of values * @param acceptableOverheadRatio an acceptable overhead ratio */ public GrowableWriter(int startBitsPerValue, int valueCount, float acceptableOverheadRatio) { this.acceptableOverheadRatio = acceptableOverheadRatio; current = PackedInts.getMutable(valueCount, startBitsPerValue, this.acceptableOverheadRatio); currentMask = mask(current.getBitsPerValue()); }
/** * Returns how many bits are required to hold values up * to and including maxValue * * @param maxValue the maximum value that should be representable. * @return the amount of bits needed to represent values from 0 to maxValue. * @see PackedInts#bitsRequired(long) */ public static int bitsRequired(long maxValue) { return roundBits(PackedInts.bitsRequired(maxValue)); }
/** * Fill the mutable from <code>fromIndex</code> (inclusive) to * <code>toIndex</code> (exclusive) with <code>val</code>. */ public void fill(int fromIndex, int toIndex, long val) { assert val <= maxValue(getBitsPerValue()); assert fromIndex <= toIndex; for (int i = fromIndex; i < toIndex; ++i) { set(i, val); } }
/** * Create a new {@link PagedMutable} instance. * * @param size the number of values to store. * @param pageSize the number of values per page * @param bitsPerValue the number of bits per value * @param acceptableOverheadRatio an acceptable overhead ratio */ public PagedMutable(long size, int pageSize, int bitsPerValue, float acceptableOverheadRatio) { this(size, pageSize, PackedInts.fastestFormatAndBits(pageSize, bitsPerValue, acceptableOverheadRatio)); fillPages(); }
/** Set value at <code>index</code>. */ public final void set(long index, long value) { assert index >= 0 && index < size; final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); subMutables[pageIndex].set(indexInPage, value); }
/** * Returns how many bits are required to hold values up * to and including maxValue, interpreted as an unsigned value. * * @param maxValue the maximum value that should be representable. * @return the amount of bits needed to represent values from 0 to maxValue. * @see PackedInts#unsignedBitsRequired(long) */ public static int unsignedBitsRequired(long maxValue) { return roundBits(PackedInts.unsignedBitsRequired(maxValue)); }
/** * Get a {@link Decoder}. * * @param format the format used to store packed ints * @param version the compatibility version * @param bitsPerValue the number of bits per value * @return a decoder */ public static Decoder getDecoder(Format format, int version, int bitsPerValue) { checkVersion(version); return BulkOperation.of(format, bitsPerValue); }
@Override public void set(int index, long value) { ensureCapacity(value); current.set(index, value); }
@Override public void fill(int fromIndex, int toIndex, long val) { ensureCapacity(val); current.fill(fromIndex, toIndex, val); }
/** * Sole constructor. * @param blockSize the number of values of a single block, must be a multiple of <tt>64</tt> */ public AbstractBlockPackedWriter(DataOutput out, int blockSize) { checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE); reset(out); values = new long[blockSize]; }
@Override int decodeBlock(int block, long[] dest) { final int count = super.decodeBlock(block, dest); final float average = averages[block]; for (int i = 0; i < count; ++i) { dest[i] += expected(0, average, i); } return count; }
@Override void pack(long[] values, int numValues, int block, float acceptableOverheadRatio) { final float average = numValues == 1 ? 0 : (float) (values[numValues - 1] - values[0]) / (numValues - 1); for (int i = 0; i < numValues; ++i) { values[i] -= expected(0, average, i); } super.pack(values, numValues, block, acceptableOverheadRatio); averages[block] = average; }
/** * Get an {@link Encoder}. * * @param format the format used to store packed ints * @param version the compatibility version * @param bitsPerValue the number of bits per value * @return an encoder */ public static Encoder getEncoder(Format format, int version, int bitsPerValue) { checkVersion(version); return BulkOperation.of(format, bitsPerValue); }