private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[] maxPackedValue, IndexInput in) throws IOException { for (int dim = 0; dim < numIndexDims; dim++) { int prefix = commonPrefixLengths[dim]; in.readBytes(minPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix); in.readBytes(maxPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix); } }
@Override public void readBytes(byte[] b, int offset, int len) throws IOException { main.readBytes(b, offset, len); digest.update(b, offset, len); }
/** Read this many bytes from in */ public void copy(IndexInput in, long byteCount) throws IOException { while (byteCount > 0) { int left = blockSize - upto; if (left == 0) { if (currentBlock != null) { addBlock(currentBlock); } currentBlock = new byte[blockSize]; upto = 0; left = blockSize; } if (left < byteCount) { in.readBytes(currentBlock, upto, left, false); upto = blockSize; byteCount -= left; } else { in.readBytes(currentBlock, upto, (int) byteCount, false); upto += byteCount; break; } } }
private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException { for(int dim=0;dim<numDataDims;dim++) { int prefix = in.readVInt(); commonPrefixLengths[dim] = prefix; if (prefix > 0) { in.readBytes(scratchPackedValue, dim*bytesPerDim, prefix); } //System.out.println("R: " + dim + " of " + numDims + " prefix=" + prefix); } }
private void visitRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException { for (int i = 0; i < count; ++i) { for(int dim=0;dim<numDataDims;dim++) { int prefix = commonPrefixLengths[dim]; in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix); } visitor.visit(docIDs[i], scratchPackedValue); } }
@Override public BytesRef binaryValue() throws IOException { bytesSlice.seek((long) doc * length); bytesSlice.readBytes(bytes.bytes, 0, length); return bytes; } };
SkipBuffer(IndexInput input, int length) throws IOException { super("SkipBuffer on " + input); data = new byte[length]; pointer = input.getFilePointer(); input.readBytes(data, 0, length); }
@Override protected void readInternal(byte[] b, int offset, int len) throws IOException { long start = getFilePointer(); if (start + len > length) { throw new EOFException("read past EOF: " + this); } base.seek(fileOffset + start); base.readBytes(b, offset, len, false); }
@Override public BytesRef binaryValue() throws IOException { bytesSlice.seek((long) disi.index() * length); bytesSlice.readBytes(bytes.bytes, 0, length); return bytes; } };
private void readTermBytes(int prefix, int suffix) throws IOException { builder.grow(prefix + suffix); input.readBytes(builder.bytes(), prefix, suffix); builder.setLength(prefix + suffix); }
@Override public BytesRef binaryValue() throws IOException { long startOffset = addresses.get(doc); bytes.length = (int) (addresses.get(doc + 1L) - startOffset); bytesSlice.seek(startOffset); bytesSlice.readBytes(bytes.bytes, 0, bytes.length); return bytes; } };
private BytesRef getFirstTermFromBlock(long block) throws IOException { assert block >= 0 && block <= (entry.termsDictSize - 1) >>> entry.termsDictBlockShift; final long blockAddress = blockAddresses.get(block); bytes.seek(blockAddress); term.length = bytes.readVInt(); bytes.readBytes(term.bytes, 0, term.length); return term; }
private static BytesRef readBytesRef(IndexInput in) throws IOException { int numBytes = in.readVInt(); if (numBytes < 0) { throw new CorruptIndexException("invalid bytes length: " + numBytes, in); } BytesRef bytes = new BytesRef(); bytes.length = numBytes; bytes.bytes = new byte[numBytes]; in.readBytes(bytes.bytes, 0, numBytes); return bytes; }
private BytesRef getTermFromIndex(long index) throws IOException { assert index >= 0 && index <= (entry.termsDictSize - 1) >>> entry.termsDictIndexShift; final long start = indexAddresses.get(index); term.length = (int) (indexAddresses.get(index + 1) - start); indexBytes.seek(start); indexBytes.readBytes(term.bytes, 0, term.length); return term; }
@Override public BytesRef next() throws IOException { if (++ord >= entry.termsDictSize) { return null; } if ((ord & blockMask) == 0L) { term.length = bytes.readVInt(); bytes.readBytes(term.bytes, 0, term.length); } else { final int token = Byte.toUnsignedInt(bytes.readByte()); int prefixLength = token & 0x0F; int suffixLength = 1 + (token >>> 4); if (prefixLength == 15) { prefixLength += bytes.readVInt(); } if (suffixLength == 16) { suffixLength += bytes.readVInt(); } term.length = prefixLength + suffixLength; bytes.readBytes(term.bytes, prefixLength, suffixLength); } return term; }
@Override public BytesRef binaryValue() throws IOException { final int index = disi.index(); long startOffset = addresses.get(index); bytes.length = (int) (addresses.get(index + 1L) - startOffset); bytesSlice.seek(startOffset); bytesSlice.readBytes(bytes.bytes, 0, bytes.length); return bytes; } };
@Override public int nextPosition() throws IOException { final int token = postingInput.readVInt(); pos += token >>> 1; if (storeOffsets) { startOffset = endOffset + postingInput.readVInt(); endOffset = startOffset + postingInput.readVInt(); } if ((token & 1) != 0) { payload.offset = 0; payload.length = postingInput.readVInt(); if (payload.length > payload.bytes.length) { payload.bytes = new byte[ArrayUtil.oversize(payload.length, 1)]; } postingInput.readBytes(payload.bytes, 0, payload.length); } else { payload.length = 0; } return pos; }
private void visitCompressedDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor, int compressedDim) throws IOException { // the byte at `compressedByteOffset` is compressed using run-length compression, // other suffix bytes are stored verbatim final int compressedByteOffset = compressedDim * bytesPerDim + commonPrefixLengths[compressedDim]; commonPrefixLengths[compressedDim]++; int i; for (i = 0; i < count; ) { scratchPackedValue[compressedByteOffset] = in.readByte(); final int runLen = Byte.toUnsignedInt(in.readByte()); for (int j = 0; j < runLen; ++j) { for(int dim=0;dim<numDataDims;dim++) { int prefix = commonPrefixLengths[dim]; in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix); } visitor.visit(docIDs[i+j], scratchPackedValue); } i += runLen; } if (i != count) { throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in); } }
/** Retrieves the full index header from the provided {@link IndexInput}. * This throws {@link CorruptIndexException} if this file does * not appear to be an index file. */ public static byte[] readIndexHeader(IndexInput in) throws IOException { in.seek(0); final int actualHeader = in.readInt(); if (actualHeader != CODEC_MAGIC) { throw new CorruptIndexException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC, in); } String codec = in.readString(); in.readInt(); in.seek(in.getFilePointer() + StringHelper.ID_LENGTH); int suffixLength = in.readByte() & 0xFF; byte[] bytes = new byte[headerLength(codec) + StringHelper.ID_LENGTH + 1 + suffixLength]; in.seek(0); in.readBytes(bytes, 0, bytes.length); return bytes; }
/** Retrieves the full footer from the provided {@link IndexInput}. This throws * {@link CorruptIndexException} if this file does not have a valid footer. */ public static byte[] readFooter(IndexInput in) throws IOException { if (in.length() < footerLength()) { throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in); } in.seek(in.length() - footerLength()); validateFooter(in); in.seek(in.length() - footerLength()); byte[] bytes = new byte[footerLength()]; in.readBytes(bytes, 0, bytes.length); return bytes; }