PackedWriter(PackedInts.Format format, DataOutput out, int valueCount, int bitsPerValue, int mem) { super(out, valueCount, bitsPerValue); this.format = format; encoder = BulkOperation.of(format, bitsPerValue); iterations = encoder.computeIterations(valueCount, mem); nextBlocks = new byte[iterations * encoder.byteBlockCount()]; nextValues = new long[iterations * encoder.byteValueCount()]; off = 0; written = 0; finished = false; }
@Override public LongsRef next(int count) throws IOException { assert nextValues.length >= 0; assert count > 0; assert nextValues.offset + nextValues.length <= nextValues.longs.length; nextValues.offset += nextValues.length; final int remaining = valueCount - position - 1; if (remaining <= 0) { throw new EOFException(); } count = Math.min(remaining, count); if (nextValues.offset == nextValues.longs.length) { final long remainingBlocks = format.byteCount(packedIntsVersion, remaining, bitsPerValue); final int blocksToRead = (int) Math.min(remainingBlocks, nextBlocks.length); in.readBytes(nextBlocks, 0, blocksToRead); if (blocksToRead < nextBlocks.length) { Arrays.fill(nextBlocks, blocksToRead, nextBlocks.length, (byte) 0); } bulkOperation.decode(nextBlocks, 0, nextValues.longs, 0, iterations); nextValues.offset = 0; } nextValues.length = Math.min(nextValues.longs.length - nextValues.offset, count); position += nextValues.length; return nextValues; }
final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); assert op.longBlockCount() == 1; assert op.longValueCount() == valuesPerBlock; final int blockIndex = index / valuesPerBlock; final int nblocks = (index + len) / valuesPerBlock - blockIndex; op.encode(arr, off, blocks, blockIndex, nblocks); final int diff = nblocks * valuesPerBlock; index += diff; len -= diff;
final int iterations = ramBudget / (byteBlockCount() + 8 * byteValueCount()); if (iterations == 0) { } else if ((iterations - 1) * byteValueCount() >= valueCount) { return (int) Math.ceil((double) valueCount / byteValueCount()); } else { return iterations;
/** * Get a {@link Decoder}. * * @param format the format used to store packed ints * @param version the compatibility version * @param bitsPerValue the number of bits per value * @return a decoder */ public static Decoder getDecoder(Format format, int version, int bitsPerValue) { checkVersion(version); return BulkOperation.of(format, bitsPerValue); }
PackedReaderIterator(PackedInts.Format format, int packedIntsVersion, int valueCount, int bitsPerValue, DataInput in, int mem) { super(valueCount, bitsPerValue, in); this.format = format; this.packedIntsVersion = packedIntsVersion; bulkOperation = BulkOperation.of(format, bitsPerValue); iterations = iterations(mem); assert valueCount == 0 || iterations > 0; nextBlocks = new byte[iterations * bulkOperation.byteBlockCount()]; nextValues = new LongsRef(new long[iterations * bulkOperation.byteValueCount()], 0, 0); nextValues.offset = nextValues.longs.length; position = -1; }
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) format.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); out.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
private int iterations(int mem) { int iterations = bulkOperation.computeIterations(valueCount, mem); if (packedIntsVersion < PackedInts.VERSION_BYTE_ALIGNED) { // make sure iterations is a multiple of 8 iterations = (iterations + 7) & 0xFFFFFFF8; } return iterations; }
/** * Get an {@link Encoder}. * * @param format the format used to store packed ints * @param version the compatibility version * @param bitsPerValue the number of bits per value * @return an encoder */ public static Encoder getEncoder(Format format, int version, int bitsPerValue) { checkVersion(version); return BulkOperation.of(format, bitsPerValue); }
PackedReaderIterator(PackedInts.Format format, int packedIntsVersion, int valueCount, int bitsPerValue, DataInput in, int mem) { super(valueCount, bitsPerValue, in); this.format = format; this.packedIntsVersion = packedIntsVersion; bulkOperation = BulkOperation.of(format, bitsPerValue); iterations = iterations(mem); assert valueCount == 0 || iterations > 0; nextBlocks = new byte[iterations * bulkOperation.byteBlockCount()]; nextValues = new LongsRef(new long[iterations * bulkOperation.byteValueCount()], 0, 0); nextValues.offset = nextValues.longs.length; position = -1; }
final int iterations = ramBudget / (byteBlockCount() + 8 * byteValueCount()); if (iterations == 0) { } else if ((iterations - 1) * byteValueCount() >= valueCount) { return (int) Math.ceil((double) valueCount / byteValueCount()); } else { return iterations;
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) PackedInts.Format.PACKED.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); output.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
private int iterations(int mem) { int iterations = bulkOperation.computeIterations(valueCount, mem); if (packedIntsVersion < PackedInts.VERSION_BYTE_ALIGNED) { // make sure iterations is a multiple of 8 iterations = (iterations + 7) & 0xFFFFFFF8; } return iterations; }
DirectWriter(DataOutput output, long numValues, int bitsPerValue) { this.output = output; this.numValues = numValues; this.bitsPerValue = bitsPerValue; encoder = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue); iterations = encoder.computeIterations((int) Math.min(numValues, Integer.MAX_VALUE), PackedInts.DEFAULT_BUFFER_SIZE); nextBlocks = new byte[iterations * encoder.byteBlockCount()]; nextValues = new long[iterations * encoder.byteValueCount()]; }
final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); assert op.longBlockCount() == 1; assert op.longValueCount() == valuesPerBlock; final int blockIndex = index / valuesPerBlock; final int nblocks = (index + len) / valuesPerBlock - blockIndex; op.encode(arr, off, blocks, blockIndex, nblocks); final int diff = nblocks * valuesPerBlock; index += diff; len -= diff;
final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); assert decoder.longBlockCount() == 1; assert decoder.longValueCount() == valuesPerBlock;
final int iterations = ramBudget / (byteBlockCount() + 8 * byteValueCount()); if (iterations == 0) { } else if ((iterations - 1) * byteValueCount() >= valueCount) { return (int) Math.ceil((double) valueCount / byteValueCount()); } else { return iterations;
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) PackedInts.Format.PACKED.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); output.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
@Override public LongsRef next(int count) throws IOException { assert nextValues.length >= 0; assert count > 0; assert nextValues.offset + nextValues.length <= nextValues.longs.length; nextValues.offset += nextValues.length; final int remaining = valueCount - position - 1; if (remaining <= 0) { throw new EOFException(); } count = Math.min(remaining, count); if (nextValues.offset == nextValues.longs.length) { final long remainingBlocks = format.byteCount(packedIntsVersion, remaining, bitsPerValue); final int blocksToRead = (int) Math.min(remainingBlocks, nextBlocks.length); in.readBytes(nextBlocks, 0, blocksToRead); if (blocksToRead < nextBlocks.length) { Arrays.fill(nextBlocks, blocksToRead, nextBlocks.length, (byte) 0); } bulkOperation.decode(nextBlocks, 0, nextValues.longs, 0, iterations); nextValues.offset = 0; } nextValues.length = Math.min(nextValues.longs.length - nextValues.offset, count); position += nextValues.length; return nextValues; }
PackedReaderIterator(PackedInts.Format format, int packedIntsVersion, int valueCount, int bitsPerValue, DataInput in, int mem) { super(valueCount, bitsPerValue, in); this.format = format; this.packedIntsVersion = packedIntsVersion; bulkOperation = BulkOperation.of(format, bitsPerValue); iterations = bulkOperation.computeIterations(valueCount, mem); assert valueCount == 0 || iterations > 0; nextBlocks = new byte[iterations * bulkOperation.byteBlockCount()]; nextValues = new LongsRef(new long[iterations * bulkOperation.byteValueCount()], 0, 0); nextValues.offset = nextValues.longs.length; position = -1; }