/** * Write a block of data (<code>For</code> format). * * @param data the data to write * @param encoded a buffer to use to encode data * @param out the destination output * @throws IOException If there is a low-level I/O error */ void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException { if (isAllEqual(data)) { out.writeByte((byte) ALL_VALUES_EQUAL); out.writeVInt(data[0]); return; } final int numBits = bitsRequired(data); assert numBits > 0 && numBits <= 32 : numBits; final PackedInts.Encoder encoder = encoders[numBits]; final int iters = iterations[numBits]; assert iters * encoder.byteValueCount() >= BLOCK_SIZE; final int encodedSize = encodedSizes[numBits]; assert iters * encoder.byteBlockCount() >= encodedSize; out.writeByte((byte) numBits); encoder.encode(data, 0, encoded, 0, iters); out.writeBytes(encoded, encodedSize); }
/** * Create a new {@link ForUtil} instance and save state into <code>out</code>. */ ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits( BLOCK_SIZE, bpv, acceptableOverheadRatio); assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue); assert formatAndBits.bitsPerValue <= 32; encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); encoders[bpv] = PackedInts.getEncoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); decoders[bpv] = PackedInts.getDecoder( formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1)); } }
private void refillDocs() throws IOException { final int left = docFreq - docUpto; assert left > 0; if (left >= BLOCK_SIZE) { forUtil.readBlock(docIn, encoded, docDeltaBuffer); if (indexHasFreq) { if (needsFreq) { forUtil.readBlock(docIn, encoded, freqBuffer); } else { forUtil.skipBlock(docIn); // skip over freqs } } } else if (docFreq == 1) { docDeltaBuffer[0] = singletonDocID; freqBuffer[0] = (int) totalTermFreq; } else { // Read vInts: readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, indexHasFreq); } docBufferUpto = 0; }
private void refillDocs() throws IOException { final int left = docFreq - docUpto; assert left > 0; if (left >= BLOCK_SIZE) { forUtil.readBlock(docIn, encoded, docDeltaBuffer); forUtil.readBlock(docIn, encoded, freqBuffer); } else if (docFreq == 1) { docDeltaBuffer[0] = singletonDocID; freqBuffer[0] = (int) totalTermFreq; } else { readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, true); } docBufferUpto = 0; }
private void skipPositions() throws IOException { // Skip positions now: int toSkip = posPendingCount - freq; final int leftInBlock = BLOCK_SIZE - posBufferUpto; if (toSkip < leftInBlock) { posBufferUpto += toSkip; } else { toSkip -= leftInBlock; while(toSkip >= BLOCK_SIZE) { assert posIn.getFilePointer() != lastPosBlockFP; forUtil.skipBlock(posIn); toSkip -= BLOCK_SIZE; } refillPositions(); posBufferUpto = toSkip; } position = 0; }
forUtil.writeBlock(docDeltaBuffer, encoded, docOut); if (writeFreqs) { forUtil.writeBlock(freqBuffer, encoded, docOut);
docIn = state.directory.openInput(docName, state.context); version = CodecUtil.checkIndexHeader(docIn, DOC_CODEC, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); forUtil = new ForUtil(docIn); CodecUtil.retrieveChecksum(docIn);
private void refillDocs() throws IOException { final int left = docFreq - docUpto; assert left > 0; if (left >= BLOCK_SIZE) { forUtil.readBlock(docIn, encoded, docDeltaBuffer); forUtil.readBlock(docIn, encoded, freqBuffer); } else if (docFreq == 1) { docDeltaBuffer[0] = singletonDocID; freqBuffer[0] = (int) totalTermFreq; } else { // Read vInts: readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, true); } docBufferUpto = 0; }
lastPosition = position; if (posBufferUpto == BLOCK_SIZE) { forUtil.writeBlock(posDeltaBuffer, encoded, posOut); forUtil.writeBlock(payloadLengthBuffer, encoded, payOut); payOut.writeVInt(payloadByteUpto); payOut.writeBytes(payloadBytes, 0, payloadByteUpto); forUtil.writeBlock(offsetStartDeltaBuffer, encoded, payOut); forUtil.writeBlock(offsetLengthBuffer, encoded, payOut);
CodecUtil.writeIndexHeader(docOut, DOC_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); forUtil = new ForUtil(acceptableOverheadRatio, docOut); if (state.fieldInfos.hasProx()) { posDeltaBuffer = new int[MAX_DATA_SIZE];
/** * Restore a {@link ForUtil} from a {@link DataInput}. */ ForUtil(DataInput in) throws IOException { int packedIntsVersion = in.readVInt(); PackedInts.checkVersion(packedIntsVersion); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final int code = in.readVInt(); final int formatId = code >>> 5; final int bitsPerValue = (code & 31) + 1; final PackedInts.Format format = PackedInts.Format.byId(formatId); assert format.isSupported(bitsPerValue); encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue); encoders[bpv] = PackedInts.getEncoder( format, packedIntsVersion, bitsPerValue); decoders[bpv] = PackedInts.getDecoder( format, packedIntsVersion, bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); } }
forUtil.readBlock(posIn, encoded, posDeltaBuffer); forUtil.readBlock(payIn, encoded, payloadLengthBuffer); int numBytes = payIn.readVInt(); } else { forUtil.skipBlock(payIn); // skip over lengths int numBytes = payIn.readVInt(); // read length of payloadBytes forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer); forUtil.readBlock(payIn, encoded, offsetLengthBuffer); } else { forUtil.skipBlock(payIn); // skip over starts forUtil.skipBlock(payIn); // skip over lengths
/** * Write a block of data (<code>For</code> format). * * @param data the data to write * @param encoded a buffer to use to encode data * @param out the destination output * @throws IOException If there is a low-level I/O error */ void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException { if (isAllEqual(data)) { out.writeByte((byte) ALL_VALUES_EQUAL); out.writeVInt(data[0]); return; } final int numBits = bitsRequired(data); assert numBits > 0 && numBits <= 32 : numBits; final PackedInts.Encoder encoder = encoders[numBits]; final int iters = iterations[numBits]; assert iters * encoder.byteValueCount() >= BLOCK_SIZE; final int encodedSize = encodedSizes[numBits]; assert iters * encoder.byteBlockCount() >= encodedSize; out.writeByte((byte) numBits); encoder.encode(data, 0, encoded, 0, iters); out.writeBytes(encoded, encodedSize); }
private void refillPositions() throws IOException { if (posIn.getFilePointer() == lastPosBlockFP) { final int count = (int) (totalTermFreq % BLOCK_SIZE); int payloadLength = 0; for(int i=0;i<count;i++) { int code = posIn.readVInt(); if (indexHasPayloads) { if ((code & 1) != 0) { payloadLength = posIn.readVInt(); } posDeltaBuffer[i] = code >>> 1; if (payloadLength != 0) { posIn.seek(posIn.getFilePointer() + payloadLength); } } else { posDeltaBuffer[i] = code; } if (indexHasOffsets) { if ((posIn.readVInt() & 1) != 0) { // offset length changed posIn.readVInt(); } } } } else { forUtil.readBlock(posIn, encoded, posDeltaBuffer); } }
private void skipPositions() throws IOException { // Skip positions now: int toSkip = posPendingCount - freq; final int leftInBlock = BLOCK_SIZE - posBufferUpto; if (toSkip < leftInBlock) { posBufferUpto += toSkip; } else { toSkip -= leftInBlock; while(toSkip >= BLOCK_SIZE) { assert posIn.getFilePointer() != lastPosBlockFP; forUtil.skipBlock(posIn); toSkip -= BLOCK_SIZE; } refillPositions(); posBufferUpto = toSkip; } position = 0; }
forUtil.writeBlock(docDeltaBuffer, encoded, docOut); if (writeFreqs) { forUtil.writeBlock(freqBuffer, encoded, docOut);
docIn = state.directory.openInput(docName, state.context); version = CodecUtil.checkIndexHeader(docIn, DOC_CODEC, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); forUtil = new ForUtil(docIn); CodecUtil.retrieveChecksum(docIn);
/** * Restore a {@link ForUtil} from a {@link DataInput}. */ ForUtil(DataInput in) throws IOException { int packedIntsVersion = in.readVInt(); PackedInts.checkVersion(packedIntsVersion); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; decoders = new PackedInts.Decoder[33]; iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { final int code = in.readVInt(); final int formatId = code >>> 5; final int bitsPerValue = (code & 31) + 1; final PackedInts.Format format = PackedInts.Format.byId(formatId); assert format.isSupported(bitsPerValue); encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue); encoders[bpv] = PackedInts.getEncoder( format, packedIntsVersion, bitsPerValue); decoders[bpv] = PackedInts.getDecoder( format, packedIntsVersion, bitsPerValue); iterations[bpv] = computeIterations(decoders[bpv]); } }
private void refillDocs() throws IOException { final int left = docFreq - docUpto; assert left > 0; if (left >= BLOCK_SIZE) { forUtil.readBlock(docIn, encoded, docDeltaBuffer); if (indexHasFreq) { if (needsFreq) { forUtil.readBlock(docIn, encoded, freqBuffer); } else { forUtil.skipBlock(docIn); // skip over freqs } } } else if (docFreq == 1) { docDeltaBuffer[0] = singletonDocID; freqBuffer[0] = (int) totalTermFreq; } else { // Read vInts: readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, indexHasFreq); } docBufferUpto = 0; }