TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) { this.docState = docWriter.docState; this.trackAllocations = trackAllocations; this.nextTermsHash = nextTermsHash; this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter(); intPool = new IntBlockPool(docWriter.intBlockAllocator); bytePool = new ByteBlockPool(docWriter.byteBlockAllocator); if (nextTermsHash != null) { // We are primary termBytePool = bytePool; nextTermsHash.termBytePool = bytePool; } }
/** * Resets the pool to its initial state reusing the first buffer. Calling * {@link IntBlockPool#nextBuffer()} is not needed after reset. */ public void reset() { this.reset(true, true); }
/** * Creates a new int slice with the given starting size and returns the slices offset in the pool. * @see SliceReader */ private int newSlice(final int size) { if (intUpto > INT_BLOCK_SIZE-size) { nextBuffer(); assert assertSliceBuffer(buffer); } final int upto = intUpto; intUpto += size; buffer[intUpto-1] = 1; return upto; }
public void add(int textStart) throws IOException { int termID = bytesHash.addByPoolOffset(textStart); if (termID >= 0) { // New posting // First time we are seeing this token since we last // flushed the hash. // Init stream slices if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i<streamCount;i++) { final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset; } postingsArray.byteStarts[termID] = intUptos[intUptoStart]; newTerm(termID); } else { termID = (-termID)-1; int intStart = postingsArray.intStarts[termID]; intUptos = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT]; intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK; addTerm(termID); } }
/** * Allocates a new slice from the given offset */ private int allocSlice(final int[] slice, final int sliceOffset) { final int level = slice[sliceOffset]; final int newLevel = NEXT_LEVEL_ARRAY[level-1]; final int newSize = LEVEL_SIZE_ARRAY[newLevel]; // Maybe allocate another block if (intUpto > INT_BLOCK_SIZE-newSize) { nextBuffer(); assert assertSliceBuffer(buffer); } final int newUpto = intUpto; final int offset = newUpto + intOffset; intUpto += newSize; // Write forwarding address at end of last slice: slice[sliceOffset] = offset; // Write new level: buffer[intUpto-1] = newLevel; return newUpto; }
void reset() { // we don't reuse so we drop everything and don't fill with 0 intPool.reset(false, false); bytePool.reset(false, false); }
TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) { this.docState = docWriter.docState; this.trackAllocations = trackAllocations; this.nextTermsHash = nextTermsHash; this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter(); intPool = new IntBlockPool(docWriter.intBlockAllocator); bytePool = new ByteBlockPool(docWriter.byteBlockAllocator); if (nextTermsHash != null) { // We are primary termBytePool = bytePool; nextTermsHash.termBytePool = bytePool; } }
public void add(int textStart) throws IOException { int termID = bytesHash.addByPoolOffset(textStart); if (termID >= 0) { // New posting // First time we are seeing this token since we last // flushed the hash. // Init stream slices if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i<streamCount;i++) { final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset; } postingsArray.byteStarts[termID] = intUptos[intUptoStart]; newTerm(termID); } else { termID = (-termID)-1; int intStart = postingsArray.intStarts[termID]; intUptos = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT]; intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK; addTerm(termID); } }
/** * Creates a new int slice with the given starting size and returns the slices offset in the pool. * @see SliceReader */ private int newSlice(final int size) { if (intUpto > INT_BLOCK_SIZE-size) { nextBuffer(); assert assertSliceBuffer(buffer); } final int upto = intUpto; intUpto += size; buffer[intUpto-1] = 1; return upto; }
/** * Resets the pool to its initial state reusing the first buffer. Calling * {@link IntBlockPool#nextBuffer()} is not needed after reset. */ public void reset() { this.reset(true, true); }
TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) { this.docState = docWriter.docState; this.trackAllocations = trackAllocations; this.nextTermsHash = nextTermsHash; this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter(); intPool = new IntBlockPool(docWriter.intBlockAllocator); bytePool = new ByteBlockPool(docWriter.byteBlockAllocator); if (nextTermsHash != null) { // We are primary termBytePool = bytePool; nextTermsHash.termBytePool = bytePool; } }
public void add(int textStart) throws IOException { int termID = bytesHash.addByPoolOffset(textStart); if (termID >= 0) { // New posting // First time we are seeing this token since we last // flushed the hash. // Init stream slices if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i<streamCount;i++) { final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset; } postingsArray.byteStarts[termID] = intUptos[intUptoStart]; newTerm(termID); } else { termID = (-termID)-1; int intStart = postingsArray.intStarts[termID]; intUptos = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT]; intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK; addTerm(termID); } }
/** * Creates a new int slice with the given starting size and returns the slices offset in the pool. * @see SliceReader */ private int newSlice(final int size) { if (intUpto > INT_BLOCK_SIZE-size) { nextBuffer(); assert assertSliceBuffer(buffer); } final int upto = intUpto; intUpto += size; buffer[intUpto-1] = 1; return upto; }
/** * Resets the pool to its initial state reusing the first buffer. Calling * {@link IntBlockPool#nextBuffer()} is not needed after reset. */ public void reset() { this.reset(true, true); }
TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) { this.docState = docWriter.docState; this.trackAllocations = trackAllocations; this.nextTermsHash = nextTermsHash; this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter(); intPool = new IntBlockPool(docWriter.intBlockAllocator); bytePool = new ByteBlockPool(docWriter.byteBlockAllocator); if (nextTermsHash != null) { // We are primary termBytePool = bytePool; nextTermsHash.termBytePool = bytePool; } }
public void add(int textStart) throws IOException { int termID = bytesHash.addByPoolOffset(textStart); if (termID >= 0) { // New posting // First time we are seeing this token since we last // flushed the hash. // Init stream slices if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i<streamCount;i++) { final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset; } postingsArray.byteStarts[termID] = intUptos[intUptoStart]; newTerm(termID); } else { termID = (-termID)-1; int intStart = postingsArray.intStarts[termID]; intUptos = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT]; intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK; addTerm(termID); } }
/** * Creates a new int slice with the given starting size and returns the slices offset in the pool. * @see SliceReader */ private int newSlice(final int size) { if (intUpto > INT_BLOCK_SIZE-size) { nextBuffer(); assert assertSliceBuffer(buffer); } final int upto = intUpto; intUpto += size; buffer[intUpto-1] = 1; return upto; }
/** * Resets the pool to its initial state reusing the first buffer. Calling * {@link IntBlockPool#nextBuffer()} is not needed after reset. */ public void reset() { this.reset(true, true); }
/** * Expert: This constructor accepts an upper limit for the number of bytes that should be reused if this instance is {@link #reset()}. * The payload storage, if used, is unaffected by maxReusuedBytes, however. * @param storeOffsets <code>true</code> if offsets should be stored * @param storePayloads <code>true</code> if payloads should be stored * @param maxReusedBytes the number of bytes that should remain in the internal memory pools after {@link #reset()} is called */ MemoryIndex(boolean storeOffsets, boolean storePayloads, long maxReusedBytes) { this.storeOffsets = storeOffsets; this.storePayloads = storePayloads; this.bytesUsed = Counter.newCounter(); final int maxBufferedByteBlocks = (int)((maxReusedBytes/2) / ByteBlockPool.BYTE_BLOCK_SIZE ); final int maxBufferedIntBlocks = (int) ((maxReusedBytes - (maxBufferedByteBlocks*ByteBlockPool.BYTE_BLOCK_SIZE))/(IntBlockPool.INT_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT)); assert (maxBufferedByteBlocks * ByteBlockPool.BYTE_BLOCK_SIZE) + (maxBufferedIntBlocks * IntBlockPool.INT_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT) <= maxReusedBytes; byteBlockPool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, maxBufferedByteBlocks, bytesUsed)); intBlockPool = new IntBlockPool(new RecyclingIntBlockAllocator(IntBlockPool.INT_BLOCK_SIZE, maxBufferedIntBlocks, bytesUsed)); postingsWriter = new SliceWriter(intBlockPool); //TODO refactor BytesRefArray to allow us to apply maxReusedBytes option payloadsBytesRefs = storePayloads ? new BytesRefArray(bytesUsed) : null; }