@Override public LongsRef next(int count) throws IOException { assert nextValues.length >= 0; assert count > 0; assert nextValues.offset + nextValues.length <= nextValues.longs.length; nextValues.offset += nextValues.length; final int remaining = valueCount - position - 1; if (remaining <= 0) { throw new EOFException(); } count = Math.min(remaining, count); if (nextValues.offset == nextValues.longs.length) { final long remainingBlocks = format.byteCount(packedIntsVersion, remaining, bitsPerValue); final int blocksToRead = (int) Math.min(remainingBlocks, nextBlocks.length); in.readBytes(nextBlocks, 0, blocksToRead); if (blocksToRead < nextBlocks.length) { Arrays.fill(nextBlocks, blocksToRead, nextBlocks.length, (byte) 0); } bulkOperation.decode(nextBlocks, 0, nextValues.longs, 0, iterations); nextValues.offset = 0; } nextValues.length = Math.min(nextValues.longs.length - nextValues.offset, count); position += nextValues.length; return nextValues; }
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) PackedInts.Format.PACKED.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); output.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) format.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); out.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
Direct8(int packedIntsVersion, DataInput in, int valueCount) throws IOException { this(valueCount); in.readBytes(values, 0, valueCount); // because packed ints have not always been byte-aligned final int remaining = (int) (PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, 8) - 1L * valueCount); for (int i = 0; i < remaining; ++i) { in.readByte(); } }
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) format.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); out.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
Direct32(int packedIntsVersion, DataInput in, int valueCount) throws IOException { this(valueCount); for (int i = 0; i < valueCount; ++i) { values[i] = in.readInt(); } // because packed ints have not always been byte-aligned final int remaining = (int) (PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, 32) - 4L * valueCount); for (int i = 0; i < remaining; ++i) { in.readByte(); } }
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) PackedInts.Format.PACKED.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); output.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) format.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); out.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
private void flush() throws IOException { encoder.encode(nextValues, 0, nextBlocks, 0, iterations); final int blockCount = (int) PackedInts.Format.PACKED.byteCount(PackedInts.VERSION_CURRENT, off, bitsPerValue); output.writeBytes(nextBlocks, blockCount); Arrays.fill(nextValues, 0L); off = 0; }
final long pointer = in.getFilePointer(); subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue)); } else { subReaders[i] = PackedInts.getReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
final long pointer = in.getFilePointer(); subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue)); } else { subReaders[i] = PackedInts.getReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
readVLong(in); final long blockBytes = PackedInts.Format.PACKED.byteCount(packedIntsVersion, blockSize, bitsPerValue); skipBytes(blockBytes); ord += blockSize;
super(valueCount, bitsPerValue); final PackedInts.Format format = PackedInts.Format.PACKED; final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array blocks = new long[longCount];
private void refill() throws IOException { final int token = in.readByte() & 0xFF; final boolean minEquals0 = (token & MIN_VALUE_EQUALS_0) != 0; final int bitsPerValue = token >>> BPV_SHIFT; if (bitsPerValue > 64) { throw new IOException("Corrupted"); } final long minValue = minEquals0 ? 0L : zigZagDecode(1L + readVLong(in)); assert minEquals0 || minValue != 0; if (bitsPerValue == 0) { Arrays.fill(values, minValue); } else { final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, packedIntsVersion, bitsPerValue); final int iterations = blockSize / decoder.byteValueCount(); final int blocksSize = iterations * decoder.byteBlockCount(); if (blocks == null || blocks.length < blocksSize) { blocks = new byte[blocksSize]; } final int valueCount = (int) Math.min(this.valueCount - ord, blockSize); final int blocksCount = (int) PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, bitsPerValue); in.readBytes(blocks, 0, blocksCount); decoder.decode(blocks, 0, values, 0, iterations); if (minValue != 0) { for (int i = 0; i < valueCount; ++i) { values[i] += minValue; } } } off = 0; }
/** * Computes how many long blocks are needed to store <code>values</code> * values of size <code>bitsPerValue</code>. */ public int longCount(int packedIntsVersion, int valueCount, int bitsPerValue) { assert bitsPerValue >= 0 && bitsPerValue <= 64 : bitsPerValue; final long byteCount = byteCount(packedIntsVersion, valueCount, bitsPerValue); assert byteCount < 8L * Integer.MAX_VALUE; if ((byteCount % 8) == 0) { return (int) (byteCount / 8); } else { return (int) (byteCount / 8 + 1); } }
protected final void writeValues(int bitsRequired) throws IOException { final PackedInts.Encoder encoder = PackedInts.getEncoder(PackedInts.Format.PACKED, PackedInts.VERSION_CURRENT, bitsRequired); final int iterations = values.length / encoder.byteValueCount(); final int blockSize = encoder.byteBlockCount() * iterations; if (blocks == null || blocks.length < blockSize) { blocks = new byte[blockSize]; } if (off < values.length) { Arrays.fill(values, off, values.length, 0L); } encoder.encode(values, 0, blocks, 0, iterations); final int blockCount = (int) PackedInts.Format.PACKED.byteCount(PackedInts.VERSION_CURRENT, off, bitsRequired); out.writeBytes(blocks, blockCount); }
Direct8(int packedIntsVersion, DataInput in, int valueCount) throws IOException { this(valueCount); in.readBytes(values, 0, valueCount); // because packed ints have not always been byte-aligned final int remaining = (int) (PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, 8) - 1L * valueCount); for (int i = 0; i < remaining; ++i) { in.readByte(); } }
/** * Compute the number of bytes required to encode a block of values that require * <code>bitsPerValue</code> bits per value with format <code>format</code>. */ private static int encodedSize(PackedInts.Format format, int packedIntsVersion, int bitsPerValue) { final long byteCount = format.byteCount(packedIntsVersion, BLOCK_SIZE, bitsPerValue); assert byteCount >= 0 && byteCount <= Integer.MAX_VALUE : byteCount; return (int) byteCount; }
Direct16(int packedIntsVersion, DataInput in, int valueCount) throws IOException { this(valueCount); for (int i = 0; i < valueCount; ++i) { values[i] = in.readShort(); } // because packed ints have not always been byte-aligned final int remaining = (int) (PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, 16) - 2L * valueCount); for (int i = 0; i < remaining; ++i) { in.readByte(); } }
Direct32(int packedIntsVersion, DataInput in, int valueCount) throws IOException { this(valueCount); for (int i = 0; i < valueCount; ++i) { values[i] = in.readInt(); } // because packed ints have not always been byte-aligned final int remaining = (int) (PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, 32) - 4L * valueCount); for (int i = 0; i < remaining; ++i) { in.readByte(); } }