private CompressingTermVectorsReader(CompressingTermVectorsReader reader) { this.fieldInfos = reader.fieldInfos; this.vectorsStream = reader.vectorsStream.clone(); this.indexReader = reader.indexReader.clone(); this.packedIntsVersion = reader.packedIntsVersion; this.compressionMode = reader.compressionMode; this.decompressor = reader.decompressor.clone(); this.chunkSize = reader.chunkSize; this.numDocs = reader.numDocs; this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0); this.version = reader.version; this.numChunks = reader.numChunks; this.numDirtyChunks = reader.numDirtyChunks; this.maxPointer = reader.maxPointer; this.closed = false; }
private int[][] readPositions(int skip, int numFields, PackedInts.Reader flags, PackedInts.Reader numTerms, int[] termFreqs, int flag, final int totalPositions, int[][] positionIndex) throws IOException { final int[][] positions = new int[numFields][]; reader.reset(vectorsStream, totalPositions); reader.skip(toSkip); positions[i] = fieldPositions; for (int j = 0; j < totalFreq; ) { final LongsRef nextPositions = reader.next(totalFreq - j); for (int k = 0; k < nextPositions.length; ++k) { fieldPositions[j++] = (int) nextPositions.longs[nextPositions.offset + k]; reader.skip(totalPositions - reader.ord()); return positions;
/** Read the next value. */ public long next() throws IOException { if (ord == valueCount) { throw new EOFException(); } if (off == blockSize) { refill(); } final long value = values[off++]; ++ord; return value; }
private void refill() throws IOException { final int token = in.readByte() & 0xFF; final boolean minEquals0 = (token & MIN_VALUE_EQUALS_0) != 0; final int bitsPerValue = token >>> BPV_SHIFT; if (bitsPerValue > 64) { throw new IOException("Corrupted"); } final long minValue = minEquals0 ? 0L : zigZagDecode(1L + readVLong(in)); assert minEquals0 || minValue != 0; if (bitsPerValue == 0) { Arrays.fill(values, minValue); } else { final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, packedIntsVersion, bitsPerValue); final int iterations = blockSize / decoder.byteValueCount(); final int blocksSize = iterations * decoder.byteBlockCount(); if (blocks == null || blocks.length < blocksSize) { blocks = new byte[blocksSize]; } final int valueCount = (int) Math.min(this.valueCount - ord, blockSize); final int blocksCount = (int) PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, bitsPerValue); in.readBytes(blocks, 0, blocksCount); decoder.decode(blocks, 0, values, 0, iterations); if (minValue != 0) { for (int i = 0; i < valueCount; ++i) { values[i] += minValue; } } } off = 0; }
/** Sole constructor. * @param blockSize the number of values of a block, must be equal to the * block size of the {@link BlockPackedWriter} which has * been used to write the stream */ public BlockPackedReaderIterator(DataInput in, int packedIntsVersion, int blockSize, long valueCount) { checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE); this.packedIntsVersion = packedIntsVersion; this.blockSize = blockSize; this.values = new long[blockSize]; this.valuesRef = new LongsRef(this.values, 0, 0); reset(in, valueCount); }
/** Read between <tt>1</tt> and <code>count</code> values. */ public LongsRef next(int count) throws IOException { assert count > 0; if (ord == valueCount) { throw new EOFException(); } if (off == blockSize) { refill(); } count = Math.min(count, blockSize - off); count = (int) Math.min(count, valueCount - ord); valuesRef.offset = off; valuesRef.length = count; off += count; ord += count; return valuesRef; }
minValues = new long[numBlocks]; minValues[i] = zigZagDecode(1L + readVLong(in));
/** Sole constructor. * @param blockSize the number of values of a block, must be equal to the * block size of the {@link BlockPackedWriter} which has * been used to write the stream */ public BlockPackedReaderIterator(DataInput in, int packedIntsVersion, int blockSize, long valueCount) { checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE); this.packedIntsVersion = packedIntsVersion; this.blockSize = blockSize; this.values = new long[blockSize]; this.valuesRef = new LongsRef(this.values, 0, 0); reset(in, valueCount); }
numFields = totalFields = vectorsStream.readVInt(); } else { reader.reset(vectorsStream, chunkDocs); int sum = 0; for (int i = docBase; i < doc; ++i) { sum += reader.next(); numFields = (int) reader.next(); sum += numFields; for (int i = doc + 1; i < docBase + chunkDocs; ++i) { sum += reader.next(); reader.reset(vectorsStream, totalTerms); reader.skip(toSkip); prefixLengths[i] = fieldPrefixLengths; for (int j = 0; j < termCount; ) { final LongsRef next = reader.next(termCount - j); for (int k = 0; k < next.length; ++k) { fieldPrefixLengths[j++] = (int) next.longs[next.offset + k]; reader.skip(totalTerms - reader.ord()); reader.reset(vectorsStream, totalTerms); docOff += reader.next(); suffixLengths[i] = fieldSuffixLengths;
chunkSize = vectorsStream.readVInt(); decompressor = compressionMode.newDecompressor(); this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);
/** Read the next value. */ public long next() throws IOException { if (ord == valueCount) { throw new EOFException(); } if (off == blockSize) { refill(); } final long value = values[off++]; ++ord; return value; }
private void refill() throws IOException { final int token = in.readByte() & 0xFF; final boolean minEquals0 = (token & MIN_VALUE_EQUALS_0) != 0; final int bitsPerValue = token >>> BPV_SHIFT; if (bitsPerValue > 64) { throw new IOException("Corrupted"); } final long minValue = minEquals0 ? 0L : zigZagDecode(1L + readVLong(in)); assert minEquals0 || minValue != 0; if (bitsPerValue == 0) { Arrays.fill(values, minValue); } else { final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, packedIntsVersion, bitsPerValue); final int iterations = blockSize / decoder.byteValueCount(); final int blocksSize = iterations * decoder.byteBlockCount(); if (blocks == null || blocks.length < blocksSize) { blocks = new byte[blocksSize]; } final int valueCount = (int) Math.min(this.valueCount - ord, blockSize); final int blocksCount = (int) PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, bitsPerValue); in.readBytes(blocks, 0, blocksCount); decoder.decode(blocks, 0, values, 0, iterations); if (minValue != 0) { for (int i = 0; i < valueCount; ++i) { values[i] += minValue; } } } off = 0; }
/** Sole constructor. * @param blockSize the number of values of a block, must be equal to the * block size of the {@link BlockPackedWriter} which has * been used to write the stream */ public BlockPackedReaderIterator(DataInput in, int packedIntsVersion, int blockSize, long valueCount) { checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE); this.packedIntsVersion = packedIntsVersion; this.blockSize = blockSize; this.values = new long[blockSize]; this.valuesRef = new LongsRef(this.values, 0, 0); reset(in, valueCount); }
private int[][] readPositions(int skip, int numFields, PackedInts.Reader flags, PackedInts.Reader numTerms, int[] termFreqs, int flag, final int totalPositions, int[][] positionIndex) throws IOException { final int[][] positions = new int[numFields][]; reader.reset(vectorsStream, totalPositions); reader.skip(toSkip); positions[i] = fieldPositions; for (int j = 0; j < totalFreq; ) { final LongsRef nextPositions = reader.next(totalFreq - j); for (int k = 0; k < nextPositions.length; ++k) { fieldPositions[j++] = (int) nextPositions.longs[nextPositions.offset + k]; reader.skip(totalPositions - reader.ord()); return positions;
private Lucene42TermVectorsReader(Lucene42TermVectorsReader reader) { this.fieldInfos = reader.fieldInfos; this.vectorsStream = reader.vectorsStream.clone(); this.indexReader = reader.indexReader.clone(); this.packedIntsVersion = reader.packedIntsVersion; this.compressionMode = reader.compressionMode; this.decompressor = reader.decompressor.clone(); this.chunkSize = reader.chunkSize; this.numDocs = reader.numDocs; this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, BLOCK_SIZE, 0); this.version = reader.version; this.closed = false; }
/** Read the next value. */ public long next() throws IOException { if (ord == valueCount) { throw new EOFException(); } if (off == blockSize) { refill(); } final long value = values[off++]; ++ord; return value; }
private void refill() throws IOException { final int token = in.readByte() & 0xFF; final boolean minEquals0 = (token & MIN_VALUE_EQUALS_0) != 0; final int bitsPerValue = token >>> BPV_SHIFT; if (bitsPerValue > 64) { throw new IOException("Corrupted"); } final long minValue = minEquals0 ? 0L : zigZagDecode(1L + readVLong(in)); assert minEquals0 || minValue != 0; if (bitsPerValue == 0) { Arrays.fill(values, minValue); } else { final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, packedIntsVersion, bitsPerValue); final int iterations = blockSize / decoder.byteValueCount(); final int blocksSize = iterations * decoder.byteBlockCount(); if (blocks == null || blocks.length < blocksSize) { blocks = new byte[blocksSize]; } final int valueCount = (int) Math.min(this.valueCount - ord, blockSize); final int blocksCount = (int) PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, bitsPerValue); in.readBytes(blocks, 0, blocksCount); decoder.decode(blocks, 0, values, 0, iterations); if (minValue != 0) { for (int i = 0; i < valueCount; ++i) { values[i] += minValue; } } } off = 0; }