@Override public byte readByte() throws IOException { final byte b = main.readByte(); digest.update(b); return b; }
private void visitCompressedDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor, int compressedDim) throws IOException { // the byte at `compressedByteOffset` is compressed using run-length compression, // other suffix bytes are stored verbatim final int compressedByteOffset = compressedDim * bytesPerDim + commonPrefixLengths[compressedDim]; commonPrefixLengths[compressedDim]++; int i; for (i = 0; i < count; ) { scratchPackedValue[compressedByteOffset] = in.readByte(); final int runLen = Byte.toUnsignedInt(in.readByte()); for (int j = 0; j < runLen; ++j) { for(int dim=0;dim<numDataDims;dim++) { int prefix = commonPrefixLengths[dim]; in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix); } visitor.visit(docIDs[i+j], scratchPackedValue); } i += runLen; } if (i != count) { throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in); } }
@Override public BytesRef next() throws IOException { if (++ord >= entry.termsDictSize) { return null; } if ((ord & blockMask) == 0L) { term.length = bytes.readVInt(); bytes.readBytes(term.bytes, 0, term.length); } else { final int token = Byte.toUnsignedInt(bytes.readByte()); int prefixLength = token & 0x0F; int suffixLength = 1 + (token >>> 4); if (prefixLength == 15) { prefixLength += bytes.readVInt(); } if (suffixLength == 16) { suffixLength += bytes.readVInt(); } term.length = prefixLength + suffixLength; bytes.readBytes(term.bytes, prefixLength, suffixLength); } return term; }
@Override public byte readByte(long pos) throws IOException { slice.seek(pos); return slice.readByte(); }
/** * Read the next block of data (<code>For</code> format). * * @param in the input to use to read data * @param encoded a buffer that can be used to store encoded data * @param decoded where to write decoded data * @throws IOException If there is a low-level I/O error */ void readBlock(IndexInput in, byte[] encoded, int[] decoded) throws IOException { final int numBits = in.readByte(); assert numBits <= 32 : numBits; if (numBits == ALL_VALUES_EQUAL) { final int value = in.readVInt(); Arrays.fill(decoded, 0, BLOCK_SIZE, value); return; } final int encodedSize = encodedSizes[numBits]; in.readBytes(encoded, 0, encodedSize); final PackedInts.Decoder decoder = decoders[numBits]; final int iters = iterations[numBits]; assert iters * decoder.byteValueCount() >= BLOCK_SIZE; decoder.decode(encoded, 0, decoded, 0, iters); }
private int readCompressedDim(IndexInput in) throws IOException { int compressedDim = in.readByte(); if (compressedDim < -1 || compressedDim >= numDataDims) { throw new CorruptIndexException("Got compressedDim="+compressedDim, in); } return compressedDim; }
/** Retrieves the full index header from the provided {@link IndexInput}. * This throws {@link CorruptIndexException} if this file does * not appear to be an index file. */ public static byte[] readIndexHeader(IndexInput in) throws IOException { in.seek(0); final int actualHeader = in.readInt(); if (actualHeader != CODEC_MAGIC) { throw new CorruptIndexException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC, in); } String codec = in.readString(); in.readInt(); in.seek(in.getFilePointer() + StringHelper.ID_LENGTH); int suffixLength = in.readByte() & 0xFF; byte[] bytes = new byte[headerLength(codec) + StringHelper.ID_LENGTH + 1 + suffixLength]; in.seek(0); in.readBytes(bytes, 0, bytes.length); return bytes; }
/** Read {@code count} integers and feed the result directly to {@link IntersectVisitor#visit(int)}. */ static void readInts(IndexInput in, int count, IntersectVisitor visitor) throws IOException { final int bpv = in.readByte(); switch (bpv) { case 0: readDeltaVInts(in, count, visitor); break; case 32: readInts32(in, count, visitor); break; case 24: readInts24(in, count, visitor); break; default: throw new IOException("Unsupported number of bits per value: " + bpv); } }
int suffixLength = in.readByte() & 0xFF; byte[] suffixBytes = new byte[suffixLength]; in.readBytes(suffixBytes, 0, suffixLength);
/** Read {@code count} integers into {@code docIDs}. */ static void readInts(IndexInput in, int count, int[] docIDs) throws IOException { final int bpv = in.readByte(); switch (bpv) { case 0: readDeltaVInts(in, count, docIDs); break; case 32: readInts32(in, count, docIDs); break; case 24: readInts24(in, count, docIDs); break; default: throw new IOException("Unsupported number of bits per value: " + bpv); } }
/** Load metadata from the given {@link IndexInput}. * @see DirectMonotonicReader#getInstance(Meta, RandomAccessInput) */ public static Meta loadMeta(IndexInput metaIn, long numValues, int blockShift) throws IOException { Meta meta = new Meta(numValues, blockShift); for (int i = 0; i < meta.numBlocks; ++i) { meta.mins[i] = metaIn.readLong(); meta.avgs[i] = Float.intBitsToFloat(metaIn.readInt()); meta.offsets[i] = metaIn.readLong(); meta.bpvs[i] = metaIn.readByte(); } return meta; }
/** * Skip the next block of data. * * @param in the input where to read data * @throws IOException If there is a low-level I/O error */ void skipBlock(IndexInput in) throws IOException { final int numBits = in.readByte(); if (numBits == ALL_VALUES_EQUAL) { in.readVInt(); return; } assert numBits > 0 && numBits <= 32 : numBits; final int encodedSize = encodedSizes[numBits]; in.seek(in.getFilePointer() + encodedSize); }
switch (roundedBits >>> 3) { case 1: rawValue = in.readByte(); break; case 2: break; case 3: rawValue = ((long)in.readShort() << 8) | (in.readByte() & 0xFFL); break; case 4: break; case 5: rawValue = ((long)in.readInt() << 8) | (in.readByte() & 0xFFL); break; case 6: break; case 7: rawValue = ((long)in.readInt() << 24) | ((in.readShort() & 0xFFFFL) << 8) | (in.readByte() & 0xFFL); break; case 8: rawValue = (in.readLong() << (8 - shiftRightBits)) | ((in.readByte() & 0xFFL) >>> shiftRightBits); shiftRightBits = 0; break;
private static void readInts24(IndexInput in, int count, int[] docIDs) throws IOException { int i; for (i = 0; i < count - 7; i += 8) { long l1 = in.readLong(); long l2 = in.readLong(); long l3 = in.readLong(); docIDs[i] = (int) (l1 >>> 40); docIDs[i+1] = (int) (l1 >>> 16) & 0xffffff; docIDs[i+2] = (int) (((l1 & 0xffff) << 8) | (l2 >>> 56)); docIDs[i+3] = (int) (l2 >>> 32) & 0xffffff; docIDs[i+4] = (int) (l2 >>> 8) & 0xffffff; docIDs[i+5] = (int) (((l2 & 0xff) << 16) | (l3 >>> 48)); docIDs[i+6] = (int) (l3 >>> 24) & 0xffffff; docIDs[i+7] = (int) l3 & 0xffffff; } for (; i < count; ++i) { docIDs[i] = (Short.toUnsignedInt(in.readShort()) << 8) | Byte.toUnsignedInt(in.readByte()); } }
private void readFields(IndexInput meta, FieldInfos infos) throws IOException { for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { FieldInfo info = infos.fieldInfo(fieldNumber); if (info == null) { throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); } else if (!info.hasNorms()) { throw new CorruptIndexException("Invalid field: " + info.name, meta); } NormsEntry entry = new NormsEntry(); entry.docsWithFieldOffset = meta.readLong(); entry.docsWithFieldLength = meta.readLong(); entry.numDocsWithField = meta.readInt(); entry.bytesPerNorm = meta.readByte(); switch (entry.bytesPerNorm) { case 0: case 1: case 2: case 4: case 8: break; default: throw new CorruptIndexException("Invalid bytesPerValue: " + entry.bytesPerNorm + ", field: " + info.name, meta); } entry.normsOffset = meta.readLong(); norms.put(info.number, entry); } }
long sumBPV = 0; for (int i = 0; i < numBlocks; ++i) { final int token = in.readByte() & 0xFF; final int bitsPerValue = token >>> BPV_SHIFT; sumBPV += bitsPerValue;
private static void readInts24(IndexInput in, int count, IntersectVisitor visitor) throws IOException { int i; for (i = 0; i < count - 7; i += 8) { long l1 = in.readLong(); long l2 = in.readLong(); long l3 = in.readLong(); visitor.visit((int) (l1 >>> 40)); visitor.visit((int) (l1 >>> 16) & 0xffffff); visitor.visit((int) (((l1 & 0xffff) << 8) | (l2 >>> 56))); visitor.visit((int) (l2 >>> 32) & 0xffffff); visitor.visit((int) (l2 >>> 8) & 0xffffff); visitor.visit((int) (((l2 & 0xff) << 16) | (l3 >>> 48))); visitor.visit((int) (l3 >>> 24) & 0xffffff); visitor.visit((int) l3 & 0xffffff); } for (; i < count; ++i) { visitor.visit((Short.toUnsignedInt(in.readShort()) << 8) | Byte.toUnsignedInt(in.readByte())); } } }
@Override public int read() throws IOException { if (counter++ >= limit) { return -1; } return (indexInput.getFilePointer() < indexInput.length()) ? (indexInput.readByte() & 0xff) : -1; }
byte b = termsIn.readByte(); if (b != 0) { throw new CorruptIndexException("Index header pretends the index has auto-prefix terms: " + b, termsIn);
final int token = vectorsStream.readByte() & 0xFF; assert token != 0; // means no term vectors, cannot happen since we checked for numFields == 0 final int bitsPerFieldNum = token & 0x1F;