DirectPackedReader(int bitsPerValue, int valueCount, IndexInput in) { super(valueCount); this.in = in; this.bitsPerValue = bitsPerValue; startPointer = in.getFilePointer(); if (bitsPerValue == 64) { valueMask = -1L; } else { valueMask = (1L << bitsPerValue) - 1; } }
DirectPacked64SingleBlockReader(int bitsPerValue, int valueCount, IndexInput in) { super(valueCount); this.in = in; this.bitsPerValue = bitsPerValue; startPointer = in.getFilePointer(); valuesPerBlock = 64 / bitsPerValue; mask = ~(~0L << bitsPerValue); }
@Override public long getFilePointer() { return main.getFilePointer(); }
SkipBuffer(IndexInput input, int length) throws IOException { super("SkipBuffer on " + input); data = new byte[length]; pointer = input.getFilePointer(); input.readBytes(data, 0, length); }
private void readBlockHeader() throws IOException { block = Short.toUnsignedInt(slice.readShort()) << 16; assert block >= 0; final int numValues = 1 + Short.toUnsignedInt(slice.readShort()); index = nextBlockIndex; nextBlockIndex = index + numValues; if (numValues <= MAX_ARRAY_LENGTH) { method = Method.SPARSE; blockEnd = slice.getFilePointer() + (numValues << 1); } else if (numValues == 65536) { method = Method.ALL; blockEnd = slice.getFilePointer(); gap = block - index - 1; } else { method = Method.DENSE; blockEnd = slice.getFilePointer() + (1 << 13); wordIndex = -1; numberOfOnes = index + 1; } }
private void skipBytes(long count) throws IOException { if (in instanceof IndexInput) { final IndexInput iin = (IndexInput) in; iin.seek(iin.getFilePointer() + count); } else { if (blocks == null) { blocks = new byte[blockSize]; } long skipped = 0; while (skipped < count) { final int toSkip = (int) Math.min(blocks.length, count - skipped); in.readBytes(blocks, 0, toSkip); skipped += toSkip; } } }
private void skipPositions() throws IOException { // Skip positions now: int toSkip = posPendingCount - freq; final int leftInBlock = BLOCK_SIZE - posBufferUpto; if (toSkip < leftInBlock) { posBufferUpto += toSkip; } else { toSkip -= leftInBlock; while(toSkip >= BLOCK_SIZE) { assert posIn.getFilePointer() != lastPosBlockFP; forUtil.skipBlock(posIn); toSkip -= BLOCK_SIZE; } refillPositions(); posBufferUpto = toSkip; } position = 0; }
/** Skips entries to the first beyond the current whose document number is * greater than or equal to <i>target</i>. Returns the current doc count. */ public int skipTo(int target) throws IOException { // walk up the levels until highest level is found that has a skip // for this target int level = 0; while (level < numberOfSkipLevels - 1 && target > skipDoc[level + 1]) { level++; } while (level >= 0) { if (target > skipDoc[level]) { if (!loadNextSkip(level)) { continue; } } else { // no more skips on this level, go down one level if (level > 0 && lastChildPointer > skipStream[level - 1].getFilePointer()) { seekChild(level - 1); } level--; } } return numSkipped[0] - skipInterval[0] - 1; }
private static void validateFooter(IndexInput in) throws IOException { long remaining = in.length() - in.getFilePointer(); long expected = footerLength(); if (remaining < expected) { throw new CorruptIndexException("misplaced codec footer (file truncated?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in); } else if (remaining > expected) { throw new CorruptIndexException("misplaced codec footer (file extended?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in); } final int magic = in.readInt(); if (magic != FOOTER_MAGIC) { throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in); } final int algorithmID = in.readInt(); if (algorithmID != 0) { throw new CorruptIndexException("codec footer mismatch: unknown algorithmID: " + algorithmID, in); } }
/** * Skip the next block of data. * * @param in the input where to read data * @throws IOException If there is a low-level I/O error */ void skipBlock(IndexInput in) throws IOException { final int numBits = in.readByte(); if (numBits == ALL_VALUES_EQUAL) { in.readVInt(); return; } assert numBits > 0 && numBits <= 32 : numBits; final int encodedSize = encodedSizes[numBits]; in.seek(in.getFilePointer() + encodedSize); }
skipPointer[i] = skipStream[0].getFilePointer(); if (toBuffer > 0) { skipStream[0].seek(skipStream[0].getFilePointer() + length); skipPointer[0] = skipStream[0].getFilePointer();
@Override public BytesRef next() { if (input.getFilePointer() < end) { try { int code = input.readVInt(); boolean newField = (code & 1) != 0; if (newField) { field = input.readString(); } int prefix = code >>> 1; int suffix = input.readVInt(); readTermBytes(prefix, suffix); return bytes; } catch (IOException e) { throw new RuntimeException(e); } } else { field = null; return null; } }
@Override boolean advanceExactWithinBlock(IndexedDISI disi, int target) throws IOException { final int targetInBlock = target & 0xFFFF; // TODO: binary search if (target == disi.doc) { return disi.exists; } for (; disi.index < disi.nextBlockIndex;) { int doc = Short.toUnsignedInt(disi.slice.readShort()); disi.index++; if (doc >= targetInBlock) { if (doc != targetInBlock) { disi.index--; disi.slice.seek(disi.slice.getFilePointer() - Short.BYTES); break; } disi.exists = true; return true; } } disi.exists = false; return false; } },
docsInBlock = bkd.readDocIDs(state.in, state.in.getFilePointer(), state.scratchDocIDs); assert docsInBlock > 0; docBlockUpto = 0;
private void refillPositions() throws IOException { if (posIn.getFilePointer() == lastPosBlockFP) { final int count = (int) (totalTermFreq % BLOCK_SIZE); int payloadLength = 0; for(int i=0;i<count;i++) { int code = posIn.readVInt(); if (indexHasPayloads) { if ((code & 1) != 0) { payloadLength = posIn.readVInt(); } posDeltaBuffer[i] = code >>> 1; if (payloadLength != 0) { posIn.seek(posIn.getFilePointer() + payloadLength); } } else { posDeltaBuffer[i] = code; } if (indexHasOffsets) { if ((posIn.readVInt() & 1) != 0) { // offset length changed posIn.readVInt(); } } } } else { forUtil.readBlock(posIn, encoded, posDeltaBuffer); } }
@Override public void markOrds(long count, LongBitSet ordBitSet) throws IOException { if (countLeft < count) { throw new IllegalStateException("only " + countLeft + " points remain, but " + count + " were requested"); } long fp = in.getFilePointer() + packedValue.length; if (singleValuePerDoc == false) { fp += Integer.BYTES; } for(long i=0;i<count;i++) { in.seek(fp); long ord; if (longOrds) { ord = in.readLong(); } else { ord = in.readInt(); } assert ordBitSet.get(ord) == false: "ord=" + ord + " i=" + i + " was seen twice from " + this; ordBitSet.set(ord); fp += bytesPerDoc; } }
toSkip -= leftInBlock; while(toSkip >= BLOCK_SIZE) { assert posIn.getFilePointer() != lastPosBlockFP; forUtil.skipBlock(posIn); payIn.seek(payIn.getFilePointer() + numBytes);
final int size = (int) Math.min(blockSize, valueCount - (long) i * blockSize); if (direct) { final long pointer = in.getFilePointer(); subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue));
/** Retrieves the full index header from the provided {@link IndexInput}. * This throws {@link CorruptIndexException} if this file does * not appear to be an index file. */ public static byte[] readIndexHeader(IndexInput in) throws IOException { in.seek(0); final int actualHeader = in.readInt(); if (actualHeader != CODEC_MAGIC) { throw new CorruptIndexException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC, in); } String codec = in.readString(); in.readInt(); in.seek(in.getFilePointer() + StringHelper.ID_LENGTH); int suffixLength = in.readByte() & 0xFF; byte[] bytes = new byte[headerLength(codec) + StringHelper.ID_LENGTH + 1 + suffixLength]; in.seek(0); in.readBytes(bytes, 0, bytes.length); return bytes; }
final int size = (int) Math.min(blockSize, valueCount - (long) i * blockSize); if (direct) { final long pointer = in.getFilePointer(); subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue); in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue));