private static void readDeltaVInts(IndexInput in, int count, int[] docIDs) throws IOException { int doc = 0; for (int i = 0; i < count; i++) { doc += in.readVInt(); docIDs[i] = doc; } }
/** * Read values that have been written using variable-length encoding instead of bit-packing. */ static void readVIntBlock(IndexInput docIn, int[] docBuffer, int[] freqBuffer, int num, boolean indexHasFreq) throws IOException { if (indexHasFreq) { for(int i=0;i<num;i++) { final int code = docIn.readVInt(); docBuffer[i] = code >>> 1; if ((code & 1) != 0) { freqBuffer[i] = 1; } else { freqBuffer[i] = docIn.readVInt(); } } } else { for(int i=0;i<num;i++) { docBuffer[i] = docIn.readVInt(); } } }
@Override public final int readVInt() throws IOException { if (5 <= (bufferLength-bufferPosition)) { byte b = buffer[bufferPosition++]; if (b >= 0) return b; int i = b & 0x7F; b = buffer[bufferPosition++]; i |= (b & 0x7F) << 7; if (b >= 0) return i; b = buffer[bufferPosition++]; i |= (b & 0x7F) << 14; if (b >= 0) return i; b = buffer[bufferPosition++]; i |= (b & 0x7F) << 21; if (b >= 0) return i; b = buffer[bufferPosition++]; // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors: i |= (b & 0x0F) << 28; if ((b & 0xF0) == 0) return i; throw new IOException("Invalid vInt detected (too many bits)"); } else { return super.readVInt(); } }
@Override public int nextPosition() throws IOException { final int token = postingInput.readVInt(); pos += token >>> 1; if (storeOffsets) { startOffset = endOffset + postingInput.readVInt(); endOffset = startOffset + postingInput.readVInt(); } if ((token & 1) != 0) { payload.offset = 0; payload.length = postingInput.readVInt(); if (payload.length > payload.bytes.length) { payload.bytes = new byte[ArrayUtil.oversize(payload.length, 1)]; } postingInput.readBytes(payload.bytes, 0, payload.length); } else { payload.length = 0; } return pos; }
private static void readDeltaVInts(IndexInput in, int count, IntersectVisitor visitor) throws IOException { int doc = 0; for (int i = 0; i < count; i++) { doc += in.readVInt(); visitor.visit(doc); } }
private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException { for(int dim=0;dim<numDataDims;dim++) { int prefix = in.readVInt(); commonPrefixLengths[dim] = prefix; if (prefix > 0) { in.readBytes(scratchPackedValue, dim*bytesPerDim, prefix); } //System.out.println("R: " + dim + " of " + numDims + " prefix=" + prefix); } }
@Override public int nextDoc() throws IOException { if (++docIt >= upto) return DocIdSetIterator.NO_MORE_DOCS; postingInput.seek(offsets[docIt]); currFreq = postingInput.readVInt(); // reset variables used in nextPosition pos = 0; endOffset = 0; return docs[docIt]; }
@Override protected int readSkipData(int level, IndexInput skipStream) throws IOException { int delta = skipStream.readVInt(); docPointer[level] += skipStream.readVLong(); if (posPointer != null) { posPointer[level] += skipStream.readVLong(); posBufferUpto[level] = skipStream.readVInt(); if (payloadByteUpto != null) { payloadByteUpto[level] = skipStream.readVInt(); } if (payPointer != null) { payPointer[level] += skipStream.readVLong(); } } return delta; } }
/** * Construct a direct {@link Reader} from an {@link IndexInput}. This method * is useful to restore data from streams which have been created using * {@link PackedInts#getWriter(DataOutput, int, int, float)}. * <p> * The returned reader will have very little memory overhead, but every call * to {@link Reader#get(int)} is likely to perform a disk seek. * * @param in the stream to read data from * @return a direct Reader * @throws IOException If there is a low-level I/O error * @lucene.internal */ public static Reader getDirectReader(IndexInput in) throws IOException { final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT); final int bitsPerValue = in.readVInt(); assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; final int valueCount = in.readVInt(); final Format format = Format.byId(in.readVInt()); return getDirectReaderNoHeader(in, format, version, valueCount, bitsPerValue); }
@Override public void init(IndexInput termsIn, SegmentReadState state) throws IOException { // Make sure we are talking to the matching postings writer CodecUtil.checkIndexHeader(termsIn, TERMS_CODEC, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); final int indexBlockSize = termsIn.readVInt(); if (indexBlockSize != BLOCK_SIZE) { throw new IllegalStateException("index-time BLOCK_SIZE (" + indexBlockSize + ") != read-time BLOCK_SIZE (" + BLOCK_SIZE + ")"); } }
private void refillPositions() throws IOException { if (posIn.getFilePointer() == lastPosBlockFP) { final int count = (int) (totalTermFreq % BLOCK_SIZE); int payloadLength = 0; for(int i=0;i<count;i++) { int code = posIn.readVInt(); if (indexHasPayloads) { if ((code & 1) != 0) { payloadLength = posIn.readVInt(); } posDeltaBuffer[i] = code >>> 1; if (payloadLength != 0) { posIn.seek(posIn.getFilePointer() + payloadLength); } } else { posDeltaBuffer[i] = code; } if (indexHasOffsets) { if ((posIn.readVInt() & 1) != 0) { // offset length changed posIn.readVInt(); } } } } else { forUtil.readBlock(posIn, encoded, posDeltaBuffer); } }
@Override public BytesRef next() throws IOException { if (++ord >= entry.termsDictSize) { return null; } if ((ord & blockMask) == 0L) { term.length = bytes.readVInt(); bytes.readBytes(term.bytes, 0, term.length); } else { final int token = Byte.toUnsignedInt(bytes.readByte()); int prefixLength = token & 0x0F; int suffixLength = 1 + (token >>> 4); if (prefixLength == 15) { prefixLength += bytes.readVInt(); } if (suffixLength == 16) { suffixLength += bytes.readVInt(); } term.length = prefixLength + suffixLength; bytes.readBytes(term.bytes, prefixLength, suffixLength); } return term; }
@Override public BytesRef next() { if (input.getFilePointer() < end) { try { int code = input.readVInt(); boolean newField = (code & 1) != 0; if (newField) { field = input.readString(); } int prefix = code >>> 1; int suffix = input.readVInt(); readTermBytes(prefix, suffix); return bytes; } catch (IOException e) { throw new RuntimeException(e); } } else { field = null; return null; } }
int readDocIDs(IndexInput in, long blockFP, int[] docIDs) throws IOException { in.seek(blockFP); // How many points are stored in this leaf cell: int count = in.readVInt(); if (version < BKDWriter.VERSION_COMPRESSED_DOC_IDS) { DocIdsWriter.readInts32(in, count, docIDs); } else { DocIdsWriter.readInts(in, count, docIDs); } return count; }
private static BytesRef readBytesRef(IndexInput in) throws IOException { int numBytes = in.readVInt(); if (numBytes < 0) { throw new CorruptIndexException("invalid bytes length: " + numBytes, in); } BytesRef bytes = new BytesRef(); bytes.length = numBytes; bytes.bytes = new byte[numBytes]; in.readBytes(bytes.bytes, 0, numBytes); return bytes; }
private BytesRef getFirstTermFromBlock(long block) throws IOException { assert block >= 0 && block <= (entry.termsDictSize - 1) >>> entry.termsDictBlockShift; final long blockAddress = blockAddresses.get(block); bytes.seek(blockAddress); term.length = bytes.readVInt(); bytes.readBytes(term.bytes, 0, term.length); return term; }
private void visitDocIDs(IndexInput in, long blockFP, IntersectVisitor visitor) throws IOException { // Leaf node in.seek(blockFP); // How many points are stored in this leaf cell: int count = in.readVInt(); // No need to call grow(), it has been called up-front if (version < BKDWriter.VERSION_COMPRESSED_DOC_IDS) { DocIdsWriter.readInts32(in, count, visitor); } else { DocIdsWriter.readInts(in, count, visitor); } }
/** * Skip the next block of data. * * @param in the input where to read data * @throws IOException If there is a low-level I/O error */ void skipBlock(IndexInput in) throws IOException { final int numBits = in.readByte(); if (numBits == ALL_VALUES_EQUAL) { in.readVInt(); return; } assert numBits > 0 && numBits <= 32 : numBits; final int encodedSize = encodedSizes[numBits]; in.seek(in.getFilePointer() + encodedSize); }
/** * Read the next block of data (<code>For</code> format). * * @param in the input to use to read data * @param encoded a buffer that can be used to store encoded data * @param decoded where to write decoded data * @throws IOException If there is a low-level I/O error */ void readBlock(IndexInput in, byte[] encoded, int[] decoded) throws IOException { final int numBits = in.readByte(); assert numBits <= 32 : numBits; if (numBits == ALL_VALUES_EQUAL) { final int value = in.readVInt(); Arrays.fill(decoded, 0, BLOCK_SIZE, value); return; } final int encodedSize = encodedSizes[numBits]; in.readBytes(encoded, 0, encodedSize); final PackedInts.Decoder decoder = decoders[numBits]; final int iters = iterations[numBits]; assert iters * decoder.byteValueCount() >= BLOCK_SIZE; decoder.decode(encoded, 0, decoded, 0, iters); }
@Override public SeekStatus seekCeil(BytesRef text) throws IOException { final long block = seekBlock(text); if (block == -1) { // before the first term seekExact(0L); return SeekStatus.NOT_FOUND; } final long blockAddress = blockAddresses.get(block); this.ord = block << entry.termsDictBlockShift; bytes.seek(blockAddress); term.length = bytes.readVInt(); bytes.readBytes(term.bytes, 0, term.length); while (true) { int cmp = term.compareTo(text); if (cmp == 0) { return SeekStatus.FOUND; } else if (cmp > 0) { return SeekStatus.NOT_FOUND; } if (next() == null) { return SeekStatus.END; } } }