private long estimatePointCount(IntersectState state, int nodeID, byte[] cellMinPacked, byte[] cellMaxPacked) { Relation r = state.visitor.compare(cellMinPacked, cellMaxPacked);
/** Fast path: this is called when the query box fully encompasses all cells under this node. */ private void addAll(IntersectState state, boolean grown) throws IOException { //System.out.println("R: addAll nodeID=" + nodeID); if (grown == false) { final long maxPointCount = (long) maxPointsInLeafNode * state.index.getNumLeaves(); if (maxPointCount <= Integer.MAX_VALUE) { // could be >MAX_VALUE if there are more than 2B points in total state.visitor.grow((int) maxPointCount); grown = true; } } if (state.index.isLeafNode()) { assert grown; //System.out.println("ADDALL"); if (state.index.nodeExists()) { visitDocIDs(state.in, state.index.getLeafBlockFP(), state.visitor); } // TODO: we can assert that the first value here in fact matches what the index claimed? } else { state.index.pushLeft(); addAll(state, grown); state.index.pop(); state.index.pushRight(); addAll(state, grown); state.index.pop(); } }
private void visitCompressedDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor, int compressedDim) throws IOException { // the byte at `compressedByteOffset` is compressed using run-length compression, // other suffix bytes are stored verbatim final int compressedByteOffset = compressedDim * bytesPerDim + commonPrefixLengths[compressedDim]; commonPrefixLengths[compressedDim]++; int i; for (i = 0; i < count; ) { scratchPackedValue[compressedByteOffset] = in.readByte(); final int runLen = Byte.toUnsignedInt(in.readByte()); for (int j = 0; j < runLen; ++j) { for(int dim=0;dim<numDataDims;dim++) { int prefix = commonPrefixLengths[dim]; in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix); } visitor.visit(docIDs[i+j], scratchPackedValue); } i += runLen; } if (i != count) { throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in); } }
/** Used to track all state for a single call to {@link #intersect}. */ public static final class IntersectState { final IndexInput in; final int[] scratchDocIDs; final byte[] scratchDataPackedValue, scratchMinIndexPackedValue, scratchMaxIndexPackedValue; final int[] commonPrefixLengths; final IntersectVisitor visitor; public final IndexTree index; public IntersectState(IndexInput in, int numDims, int packedBytesLength, int packedIndexBytesLength, int maxPointsInLeafNode, IntersectVisitor visitor, IndexTree indexVisitor) { this.in = in; this.visitor = visitor; this.commonPrefixLengths = new int[numDims]; this.scratchDocIDs = new int[maxPointsInLeafNode]; this.scratchDataPackedValue = new byte[packedBytesLength]; this.scratchMinIndexPackedValue = new byte[packedIndexBytesLength]; this.scratchMaxIndexPackedValue = new byte[packedIndexBytesLength]; this.index = indexVisitor; } }
private void visitCompressedDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor, int compressedDim) throws IOException { // the byte at `compressedByteOffset` is compressed using run-length compression, // other suffix bytes are stored verbatim final int compressedByteOffset = compressedDim * bytesPerDim + commonPrefixLengths[compressedDim]; commonPrefixLengths[compressedDim]++; int i; for (i = 0; i < count; ) { scratchPackedValue[compressedByteOffset] = in.readByte(); final int runLen = Byte.toUnsignedInt(in.readByte()); for (int j = 0; j < runLen; ++j) { for(int dim=0;dim<numDataDims;dim++) { int prefix = commonPrefixLengths[dim]; in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix); } visitor.visit(docIDs[i+j], scratchPackedValue); } i += runLen; } if (i != count) { throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in); } }
private static void readInts24(IndexInput in, int count, IntersectVisitor visitor) throws IOException { int i; for (i = 0; i < count - 7; i += 8) { long l1 = in.readLong(); long l2 = in.readLong(); long l3 = in.readLong(); visitor.visit((int) (l1 >>> 40)); visitor.visit((int) (l1 >>> 16) & 0xffffff); visitor.visit((int) (((l1 & 0xffff) << 8) | (l2 >>> 56))); visitor.visit((int) (l2 >>> 32) & 0xffffff); visitor.visit((int) (l2 >>> 8) & 0xffffff); visitor.visit((int) (((l2 & 0xff) << 16) | (l3 >>> 48))); visitor.visit((int) (l3 >>> 24) & 0xffffff); visitor.visit((int) l3 & 0xffffff); } for (; i < count; ++i) { visitor.visit((Short.toUnsignedInt(in.readShort()) << 8) | Byte.toUnsignedInt(in.readByte())); } } }
/** Used to track all state for a single call to {@link #intersect}. */ public static final class IntersectState { final IndexInput in; final int[] scratchDocIDs; final byte[] scratchPackedValue; final int[] commonPrefixLengths; final IntersectVisitor visitor; public IntersectState(IndexInput in, int numDims, int packedBytesLength, int maxPointsInLeafNode, IntersectVisitor visitor) { this.in = in; this.visitor = visitor; this.commonPrefixLengths = new int[numDims]; this.scratchDocIDs = new int[maxPointsInLeafNode]; this.scratchPackedValue = new byte[packedBytesLength]; } }
void visitDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException { visitor.grow(count); // NOTE: we don't do prefix coding, so we ignore commonPrefixLengths assert scratchPackedValue.length == packedBytesLength; BytesRefBuilder scratch = new BytesRefBuilder(); for(int i=0;i<count;i++) { readLine(in, scratch); assert startsWith(scratch, BLOCK_VALUE); BytesRef br = SimpleTextUtil.fromBytesRefString(stripPrefix(scratch, BLOCK_VALUE)); assert br.length == packedBytesLength; System.arraycopy(br.bytes, br.offset, scratchPackedValue, 0, packedBytesLength); visitor.visit(docIDs[i], scratchPackedValue); } }
@Override public void visit(int docID, byte[] packedValue) throws IOException { visitor.visit(docMap.oldToNew(docID), packedValue); }
@Override public void visit(int docID, byte[] packedValue) throws IOException { visitor.visit(docMap.oldToNew(docID), packedValue); }
private void visitRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException { for (int i = 0; i < count; ++i) { for(int dim=0;dim<numDataDims;dim++) { int prefix = commonPrefixLengths[dim]; in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix); } visitor.visit(docIDs[i], scratchPackedValue); } }
@Override public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { return visitor.compare(minPackedValue, maxPackedValue); } });
@Override public void visit(int docID) throws IOException { visitor.visit(docMap.oldToNew(docID)); }
@Override public void visit(int docID) throws IOException { visitor.visit(docMap.oldToNew(docID)); }
static void readInts32(IndexInput in, int count, IntersectVisitor visitor) throws IOException { for (int i = 0; i < count; i++) { visitor.visit(in.readInt()); } }
private static void readDeltaVInts(IndexInput in, int count, IntersectVisitor visitor) throws IOException { int doc = 0; for (int i = 0; i < count; i++) { doc += in.readVInt(); visitor.visit(doc); } }
@Override public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { return visitor.compare(minPackedValue, maxPackedValue); } });
void visitDocIDs(IndexInput in, long blockFP, IntersectVisitor visitor) throws IOException { BytesRefBuilder scratch = new BytesRefBuilder(); in.seek(blockFP); readLine(in, scratch); int count = parseInt(scratch, BLOCK_COUNT); visitor.grow(count); for(int i=0;i<count;i++) { readLine(in, scratch); visitor.visit(parseInt(scratch, BLOCK_DOC_ID)); } }