/** * Allocates a new slice with the given size. * @see ByteBlockPool#FIRST_LEVEL_SIZE */ public int newSlice(final int size) { if (byteUpto > BYTE_BLOCK_SIZE-size) nextBuffer(); final int upto = byteUpto; byteUpto += size; buffer[byteUpto-1] = 16; return upto; }
/** * Creates a new byte slice with the given starting size and * returns the slices offset in the pool. */ public int allocSlice(final byte[] slice, final int upto) { final int level = slice[upto] & 15; final int newLevel = NEXT_LEVEL_ARRAY[level]; final int newSize = LEVEL_SIZE_ARRAY[newLevel]; // Maybe allocate another block if (byteUpto > BYTE_BLOCK_SIZE-newSize) { nextBuffer(); } final int newUpto = byteUpto; final int offset = newUpto + byteOffset; byteUpto += newSize; // Copy forward the past 3 bytes (which we are about // to overwrite with the forwarding address): buffer[newUpto] = slice[upto-3]; buffer[newUpto+1] = slice[upto-2]; buffer[newUpto+2] = slice[upto-1]; // Write forwarding address at end of last slice: slice[upto-3] = (byte) (offset >>> 24); slice[upto-2] = (byte) (offset >>> 16); slice[upto-1] = (byte) (offset >>> 8); slice[upto] = (byte) offset; // Write new level: buffer[byteUpto-1] = (byte) (16|newLevel); return newUpto+3; }
/** * Appends the bytes in the provided {@link BytesRef} at * the current position. */ public void append(final BytesRef bytes) { int bytesLeft = bytes.length; int offset = bytes.offset; while (bytesLeft > 0) { int bufferLeft = BYTE_BLOCK_SIZE - byteUpto; if (bytesLeft < bufferLeft) { // fits within current buffer System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesLeft); byteUpto += bytesLeft; break; } else { // fill up this buffer and move to next one if (bufferLeft > 0) { System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bufferLeft); } nextBuffer(); bytesLeft -= bufferLeft; offset += bufferLeft; } } }
/** * Creates a new {@link BytesRefArray} with a counter to track allocated bytes */ public BytesRefArray(Counter bytesUsed) { this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator( bytesUsed)); pool.nextBuffer(); bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER * Integer.BYTES); this.bytesUsed = bytesUsed; }
+ (BYTE_BLOCK_SIZE - 2) + " in length; got " + bytes.length); pool.nextBuffer();
public void add(int textStart) throws IOException { int termID = bytesHash.addByPoolOffset(textStart); if (termID >= 0) { // New posting // First time we are seeing this token since we last // flushed the hash. // Init stream slices if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i<streamCount;i++) { final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset; } postingsArray.byteStarts[termID] = intUptos[intUptoStart]; newTerm(termID); } else { termID = (-termID)-1; int intStart = postingsArray.intStarts[termID]; intUptos = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT]; intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK; addTerm(termID); } }
bytePool.nextBuffer();
/** * Allocates a new slice with the given size. * @see ByteBlockPool#FIRST_LEVEL_SIZE */ public int newSlice(final int size) { if (byteUpto > BYTE_BLOCK_SIZE-size) nextBuffer(); final int upto = byteUpto; byteUpto += size; buffer[byteUpto-1] = 16; return upto; }
/** * Allocates a new slice with the given size. * @see ByteBlockPool#FIRST_LEVEL_SIZE */ public int newSlice(final int size) { if (byteUpto > BYTE_BLOCK_SIZE-size) nextBuffer(); final int upto = byteUpto; byteUpto += size; buffer[byteUpto-1] = 16; return upto; }
/** * Allocates a new slice with the given size. * @see ByteBlockPool#FIRST_LEVEL_SIZE */ public int newSlice(final int size) { if (byteUpto > BYTE_BLOCK_SIZE-size) nextBuffer(); final int upto = byteUpto; byteUpto += size; buffer[byteUpto-1] = 16; return upto; }
/** * Creates a new byte slice with the given starting size and * returns the slices offset in the pool. */ public int allocSlice(final byte[] slice, final int upto) { final int level = slice[upto] & 15; final int newLevel = NEXT_LEVEL_ARRAY[level]; final int newSize = LEVEL_SIZE_ARRAY[newLevel]; // Maybe allocate another block if (byteUpto > BYTE_BLOCK_SIZE-newSize) { nextBuffer(); } final int newUpto = byteUpto; final int offset = newUpto + byteOffset; byteUpto += newSize; // Copy forward the past 3 bytes (which we are about // to overwrite with the forwarding address): buffer[newUpto] = slice[upto-3]; buffer[newUpto+1] = slice[upto-2]; buffer[newUpto+2] = slice[upto-1]; // Write forwarding address at end of last slice: slice[upto-3] = (byte) (offset >>> 24); slice[upto-2] = (byte) (offset >>> 16); slice[upto-1] = (byte) (offset >>> 8); slice[upto] = (byte) offset; // Write new level: buffer[byteUpto-1] = (byte) (16|newLevel); return newUpto+3; }
/** * Creates a new byte slice with the given starting size and * returns the slices offset in the pool. */ public int allocSlice(final byte[] slice, final int upto) { final int level = slice[upto] & 15; final int newLevel = NEXT_LEVEL_ARRAY[level]; final int newSize = LEVEL_SIZE_ARRAY[newLevel]; // Maybe allocate another block if (byteUpto > BYTE_BLOCK_SIZE-newSize) { nextBuffer(); } final int newUpto = byteUpto; final int offset = newUpto + byteOffset; byteUpto += newSize; // Copy forward the past 3 bytes (which we are about // to overwrite with the forwarding address): buffer[newUpto] = slice[upto-3]; buffer[newUpto+1] = slice[upto-2]; buffer[newUpto+2] = slice[upto-1]; // Write forwarding address at end of last slice: slice[upto-3] = (byte) (offset >>> 24); slice[upto-2] = (byte) (offset >>> 16); slice[upto-1] = (byte) (offset >>> 8); slice[upto] = (byte) offset; // Write new level: buffer[byteUpto-1] = (byte) (16|newLevel); return newUpto+3; }
/** * Appends the bytes in the provided {@link BytesRef} at * the current position. */ public void append(final BytesRef bytes) { int bytesLeft = bytes.length; int offset = bytes.offset; while (bytesLeft > 0) { int bufferLeft = BYTE_BLOCK_SIZE - byteUpto; if (bytesLeft < bufferLeft) { // fits within current buffer System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesLeft); byteUpto += bytesLeft; break; } else { // fill up this buffer and move to next one if (bufferLeft > 0) { System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bufferLeft); } nextBuffer(); bytesLeft -= bufferLeft; offset += bufferLeft; } } }
/** * Appends the bytes in the provided {@link BytesRef} at * the current position. */ public void append(final BytesRef bytes) { int length = bytes.length; if (length == 0) { return; } int offset = bytes.offset; int overflow = (length + byteUpto) - BYTE_BLOCK_SIZE; do { if (overflow <= 0) { System.arraycopy(bytes.bytes, offset, buffer, byteUpto, length); byteUpto += length; break; } else { final int bytesToCopy = length-overflow; if (bytesToCopy > 0) { System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesToCopy); offset += bytesToCopy; length -= bytesToCopy; } nextBuffer(); overflow = overflow - BYTE_BLOCK_SIZE; } } while(true); }
/** * Appends the bytes in the provided {@link BytesRef} at * the current position. */ public void append(final BytesRef bytes) { int length = bytes.length; if (length == 0) { return; } int offset = bytes.offset; int overflow = (length + byteUpto) - BYTE_BLOCK_SIZE; do { if (overflow <= 0) { System.arraycopy(bytes.bytes, offset, buffer, byteUpto, length); byteUpto += length; break; } else { final int bytesToCopy = length-overflow; if (bytesToCopy > 0) { System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesToCopy); offset += bytesToCopy; length -= bytesToCopy; } nextBuffer(); overflow = overflow - BYTE_BLOCK_SIZE; } } while(true); }
/** * Creates a new {@link BytesRefList} */ public BytesRefList() { this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator( bytesUsed)); pool.nextBuffer(); bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_INT); }
/** * Creates a new {@link BytesRefArray} with a counter to track allocated bytes */ public BytesRefArray(Counter bytesUsed) { this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator( bytesUsed)); pool.nextBuffer(); bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_INT); this.bytesUsed = bytesUsed; }
/** * Creates a new {@link BytesRefArray} with a counter to track allocated bytes */ public BytesRefArray(Counter bytesUsed) { this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator( bytesUsed)); pool.nextBuffer(); bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_INT); this.bytesUsed = bytesUsed; }
/** * Creates a new {@link BytesRefArray} with a counter to track allocated bytes */ public BytesRefArray(Counter bytesUsed) { this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator( bytesUsed)); pool.nextBuffer(); bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER * Integer.BYTES); this.bytesUsed = bytesUsed; }
public void add(int textStart) throws IOException { int termID = bytesHash.addByPoolOffset(textStart); if (termID >= 0) { // New posting // First time we are seeing this token since we last // flushed the hash. // Init stream slices if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i<streamCount;i++) { final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset; } postingsArray.byteStarts[termID] = intUptos[intUptoStart]; newTerm(termID); } else { termID = (-termID)-1; int intStart = postingsArray.intStarts[termID]; intUptos = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT]; intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK; addTerm(termID); } }