TermsDict(TermsDictEntry entry, IndexInput data) throws IOException { this.entry = entry; RandomAccessInput addressesSlice = data.randomAccessSlice(entry.termsAddressesOffset, entry.termsAddressesLength); blockAddresses = DirectMonotonicReader.getInstance(entry.termsAddressesMeta, addressesSlice); bytes = data.slice("terms", entry.termsDataOffset, entry.termsDataLength); blockMask = (1L << entry.termsDictBlockShift) - 1; RandomAccessInput indexAddressesSlice = data.randomAccessSlice(entry.termsIndexAddressesOffset, entry.termsIndexAddressesLength); indexAddresses = DirectMonotonicReader.getInstance(entry.termsIndexAddressesMeta, indexAddressesSlice); indexBytes = data.slice("terms-index", entry.termsIndexOffset, entry.termsIndexLength); term = new BytesRef(entry.maxTermLength); }
private SortedNumericEntry readSortedNumeric(ChecksumIndexInput meta) throws IOException { SortedNumericEntry entry = new SortedNumericEntry(); readNumeric(meta, entry); entry.numDocsWithField = meta.readInt(); if (entry.numDocsWithField != entry.numValues) { entry.addressesOffset = meta.readLong(); final int blockShift = meta.readVInt(); entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift); ramBytesUsed += entry.addressesMeta.ramBytesUsed(); entry.addressesLength = meta.readLong(); } return entry; }
final LongValues addresses = DirectMonotonicReader.getInstance(entry.addressesMeta, addressesData); return new DenseBinaryDocValues(maxDoc) { final BytesRef bytes = new BytesRef(new byte[entry.maxLength], 0, entry.maxLength); final LongValues addresses = DirectMonotonicReader.getInstance(entry.addressesMeta, addressesData); return new SparseBinaryDocValues(disi) { final BytesRef bytes = new BytesRef(new byte[entry.maxLength], 0, entry.maxLength);
private static void readTermDict(ChecksumIndexInput meta, TermsDictEntry entry) throws IOException { entry.termsDictSize = meta.readVLong(); entry.termsDictBlockShift = meta.readInt(); final int blockShift = meta.readInt(); final long addressesSize = (entry.termsDictSize + (1L << entry.termsDictBlockShift) - 1) >>> entry.termsDictBlockShift; entry.termsAddressesMeta = DirectMonotonicReader.loadMeta(meta, addressesSize, blockShift); entry.maxTermLength = meta.readInt(); entry.termsDataOffset = meta.readLong(); entry.termsDataLength = meta.readLong(); entry.termsAddressesOffset = meta.readLong(); entry.termsAddressesLength = meta.readLong(); entry.termsDictIndexShift = meta.readInt(); final long indexSize = (entry.termsDictSize + (1L << entry.termsDictIndexShift) - 1) >>> entry.termsDictIndexShift; entry.termsIndexAddressesMeta = DirectMonotonicReader.loadMeta(meta, 1 + indexSize, blockShift); entry.termsIndexOffset = meta.readLong(); entry.termsIndexLength = meta.readLong(); entry.termsIndexAddressesOffset = meta.readLong(); entry.termsIndexAddressesLength = meta.readLong(); }
final LongValues addresses = DirectMonotonicReader.getInstance(entry.addressesMeta, addressesInput);
private BinaryEntry readBinary(ChecksumIndexInput meta) throws IOException { BinaryEntry entry = new BinaryEntry(); entry.dataOffset = meta.readLong(); entry.dataLength = meta.readLong(); entry.docsWithFieldOffset = meta.readLong(); entry.docsWithFieldLength = meta.readLong(); entry.numDocsWithField = meta.readInt(); entry.minLength = meta.readInt(); entry.maxLength = meta.readInt(); if (entry.minLength < entry.maxLength) { entry.addressesOffset = meta.readLong(); final int blockShift = meta.readVInt(); entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1L, blockShift); ramBytesUsed += entry.addressesMeta.ramBytesUsed(); entry.addressesLength = meta.readLong(); } return entry; }
final LongValues addresses = DirectMonotonicReader.getInstance(entry.addressesMeta, addressesInput);
private SortedSetEntry readSortedSet(ChecksumIndexInput meta) throws IOException { SortedSetEntry entry = new SortedSetEntry(); byte multiValued = meta.readByte(); switch (multiValued) { case 0: // singlevalued entry.singleValueEntry = readSorted(meta); return entry; case 1: // multivalued break; default: throw new CorruptIndexException("Invalid multiValued flag: " + multiValued, meta); } entry.docsWithFieldOffset = meta.readLong(); entry.docsWithFieldLength = meta.readLong(); entry.bitsPerValue = meta.readByte(); entry.ordsOffset = meta.readLong(); entry.ordsLength = meta.readLong(); entry.numDocsWithField = meta.readInt(); entry.addressesOffset = meta.readLong(); final int blockShift = meta.readVInt(); entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift); ramBytesUsed += entry.addressesMeta.ramBytesUsed(); entry.addressesLength = meta.readLong(); readTermDict(meta, entry); return entry; }
/** returns an address instance for sortedset ordinal lists */ private LongValues getOrdIndexInstance(FieldInfo field, NumericEntry entry) throws IOException { RandomAccessInput data = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset); return DirectMonotonicReader.getInstance(entry.monotonicMeta, data); }
private SortedNumericEntry readSortedNumeric(ChecksumIndexInput meta) throws IOException { SortedNumericEntry entry = new SortedNumericEntry(); readNumeric(meta, entry); entry.numDocsWithField = meta.readInt(); if (entry.numDocsWithField != entry.numValues) { entry.addressesOffset = meta.readLong(); final int blockShift = meta.readVInt(); entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift); ramBytesUsed += entry.addressesMeta.ramBytesUsed(); entry.addressesLength = meta.readLong(); } return entry; }
/** returns an address instance for sortedset ordinal lists */ private LongValues getOrdIndexInstance(FieldInfo field, NumericEntry entry) throws IOException { RandomAccessInput data = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset); return DirectMonotonicReader.getInstance(entry.monotonicMeta, data); }
private static void readTermDict(ChecksumIndexInput meta, TermsDictEntry entry) throws IOException { entry.termsDictSize = meta.readVLong(); entry.termsDictBlockShift = meta.readInt(); final int blockShift = meta.readInt(); final long addressesSize = (entry.termsDictSize + (1L << entry.termsDictBlockShift) - 1) >>> entry.termsDictBlockShift; entry.termsAddressesMeta = DirectMonotonicReader.loadMeta(meta, addressesSize, blockShift); entry.maxTermLength = meta.readInt(); entry.termsDataOffset = meta.readLong(); entry.termsDataLength = meta.readLong(); entry.termsAddressesOffset = meta.readLong(); entry.termsAddressesLength = meta.readLong(); entry.termsDictIndexShift = meta.readInt(); final long indexSize = (entry.termsDictSize + (1L << entry.termsDictIndexShift) - 1) >>> entry.termsDictIndexShift; entry.termsIndexAddressesMeta = DirectMonotonicReader.loadMeta(meta, 1 + indexSize, blockShift); entry.termsIndexOffset = meta.readLong(); entry.termsIndexLength = meta.readLong(); entry.termsIndexAddressesOffset = meta.readLong(); entry.termsIndexAddressesLength = meta.readLong(); }
/** returns an address instance for sortedset ordinal lists */ private LongValues getOrdIndexInstance(FieldInfo field, NumericEntry entry) throws IOException { RandomAccessInput data = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset); return DirectMonotonicReader.getInstance(entry.monotonicMeta, data); }
private BinaryEntry readBinaryEntry(FieldInfo info, IndexInput meta) throws IOException { BinaryEntry entry = new BinaryEntry(); entry.format = meta.readVInt(); entry.missingOffset = meta.readLong(); entry.minLength = meta.readVInt(); entry.maxLength = meta.readVInt(); entry.count = meta.readVLong(); entry.offset = meta.readLong(); switch(entry.format) { case BINARY_FIXED_UNCOMPRESSED: break; case BINARY_PREFIX_COMPRESSED: entry.addressesOffset = meta.readLong(); entry.packedIntsVersion = meta.readVInt(); entry.blockSize = meta.readVInt(); entry.reverseIndexOffset = meta.readLong(); break; case BINARY_VARIABLE_UNCOMPRESSED: entry.addressesOffset = meta.readLong(); final int blockShift = meta.readVInt(); entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.count + 1, blockShift); ramBytesUsed.addAndGet(entry.addressesMeta.ramBytesUsed()); directAddressesMeta.put(info.name, entry.addressesMeta); entry.addressesEndOffset = meta.readLong(); break; default: throw new CorruptIndexException("Unknown format: " + entry.format, meta); } return entry; }
private SparseBits getSparseLiveBits(NumericEntry entry) throws IOException { final RandomAccessInput docIdsData = this.data.randomAccessSlice(entry.missingOffset, entry.offset - entry.missingOffset); final LongValues docIDs = DirectMonotonicReader.getInstance(entry.monotonicMeta, docIdsData); return new SparseBits(maxDoc, entry.numDocsWithValue, docIDs); }
private BinaryEntry readBinaryEntry(FieldInfo info, IndexInput meta) throws IOException { BinaryEntry entry = new BinaryEntry(); entry.format = meta.readVInt(); entry.missingOffset = meta.readLong(); entry.minLength = meta.readVInt(); entry.maxLength = meta.readVInt(); entry.count = meta.readVLong(); entry.offset = meta.readLong(); switch(entry.format) { case BINARY_FIXED_UNCOMPRESSED: break; case BINARY_PREFIX_COMPRESSED: entry.addressesOffset = meta.readLong(); entry.packedIntsVersion = meta.readVInt(); entry.blockSize = meta.readVInt(); entry.reverseIndexOffset = meta.readLong(); break; case BINARY_VARIABLE_UNCOMPRESSED: entry.addressesOffset = meta.readLong(); final int blockShift = meta.readVInt(); entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.count + 1, blockShift); ramBytesUsed.addAndGet(entry.addressesMeta.ramBytesUsed()); directAddressesMeta.put(info.name, entry.addressesMeta); entry.addressesEndOffset = meta.readLong(); break; default: throw new CorruptIndexException("Unknown format: " + entry.format, meta); } return entry; }
private SparseBits getSparseLiveBits(NumericEntry entry) throws IOException { final RandomAccessInput docIdsData = this.data.randomAccessSlice(entry.missingOffset, entry.offset - entry.missingOffset); final LongValues docIDs = DirectMonotonicReader.getInstance(entry.monotonicMeta, docIdsData); return new SparseBits(maxDoc, entry.numDocsWithValue, docIDs); }
private BinaryEntry readBinaryEntry(FieldInfo info, IndexInput meta) throws IOException { BinaryEntry entry = new BinaryEntry(); entry.format = meta.readVInt(); entry.missingOffset = meta.readLong(); entry.minLength = meta.readVInt(); entry.maxLength = meta.readVInt(); entry.count = meta.readVLong(); entry.offset = meta.readLong(); switch(entry.format) { case BINARY_FIXED_UNCOMPRESSED: break; case BINARY_PREFIX_COMPRESSED: entry.addressesOffset = meta.readLong(); entry.packedIntsVersion = meta.readVInt(); entry.blockSize = meta.readVInt(); entry.reverseIndexOffset = meta.readLong(); break; case BINARY_VARIABLE_UNCOMPRESSED: entry.addressesOffset = meta.readLong(); final int blockShift = meta.readVInt(); entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.count + 1, blockShift); ramBytesUsed.addAndGet(entry.addressesMeta.ramBytesUsed()); directAddressesMeta.put(info.name, entry.addressesMeta); entry.addressesEndOffset = meta.readLong(); break; default: throw new CorruptIndexException("Unknown format: " + entry.format, meta); } return entry; }
private SparseNumericDocValues getSparseNumericDocValues(NumericEntry entry) throws IOException { final RandomAccessInput docIdsData = this.data.randomAccessSlice(entry.missingOffset, entry.offset - entry.missingOffset); final LongValues docIDs = DirectMonotonicReader.getInstance(entry.monotonicMeta, docIdsData); final LongValues values = getNumeric(entry.nonMissingValues); // cannot be sparse return new SparseNumericDocValues(Math.toIntExact(entry.numDocsWithValue), docIDs, values); }
entry.monotonicMeta = DirectMonotonicReader.loadMeta(meta, entry.numDocsWithValue, blockShift); ramBytesUsed.addAndGet(entry.monotonicMeta.ramBytesUsed()); directAddressesMeta.put(info.name, entry.monotonicMeta); case MONOTONIC_COMPRESSED: final int blockShift = meta.readVInt(); entry.monotonicMeta = DirectMonotonicReader.loadMeta(meta, maxDoc + 1, blockShift); ramBytesUsed.addAndGet(entry.monotonicMeta.ramBytesUsed()); directAddressesMeta.put(info.name, entry.monotonicMeta);