@Override public void run() { MemStoreLAB memStoreLAB = new MemStoreLABImpl(conf); for (int i = 0; i < maxCount; i++) { memStoreLAB.copyCellInto(kv);// Try allocate size = chunkSize. Means every // allocate call will result in a new chunk } // Close MemStoreLAB so that all chunks will be tried to be put back to pool memStoreLAB.close(); } };
public void decScannerCount() { if (this.memStoreLAB != null) { this.memStoreLAB.decScannerCount(); } }
public void incScannerCount() { if (this.memStoreLAB != null) { this.memStoreLAB.incScannerCount(); } }
/** * If the segment has a memory allocator the cell is being cloned to this space, and returned; * otherwise the given cell is returned * * When a cell's size is too big (bigger than maxAlloc), it is not allocated on MSLAB. * Since the process of flattening to CellChunkMap assumes that all cells * are allocated on MSLAB, during this process, the input parameter * forceCloneOfBigCell is set to 'true' and the cell is copied into MSLAB. * * @return either the given cell or its clone */ public Cell maybeCloneWithAllocator(Cell cell, boolean forceCloneOfBigCell) { if (this.memStoreLAB == null) { return cell; } Cell cellFromMslab; if (forceCloneOfBigCell) { cellFromMslab = this.memStoreLAB.forceCopyOfBigCellInto(cell); } else { cellFromMslab = this.memStoreLAB.copyCellInto(cell); } return (cellFromMslab != null) ? cellFromMslab : cell; }
@Override public void doAnAction() throws Exception { int valSize = r.nextInt(3); KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]); int size = kv.getSerializedSize(); ByteBufferKeyValue newCell = (ByteBufferKeyValue) mslab.copyCellInto(kv); totalAllocated.addAndGet(size); allocsByThisThread.add(new AllocRecord(newCell.getBuffer(), newCell.getOffset(), size)); } };
private void checkAndCloseMSLABs(int openScanners) { if (openScanners == 0) { for (MemStoreLAB mslab : this.mslabs) { mslab.close(); } } }
/** * @return The increase in heap size because of this cell addition. This includes this cell POJO's * heap size itself and additional overhead because of addition on to CSLM. */ protected long heapSizeChange(Cell cell, boolean allocated) { long res = 0; if (allocated) { boolean onHeap = true; MemStoreLAB memStoreLAB = getMemStoreLAB(); if(memStoreLAB != null) { onHeap = memStoreLAB.isOnHeap(); } res += indexEntryOnHeapSize(onHeap); if(onHeap) { res += cell.heapSize(); } res = ClassSize.align(res); } return res; }
private KeyValue maybeCloneWithAllocator(KeyValue kv) { if (allocator == null) { return kv; } int len = kv.getLength(); Allocation alloc = allocator.allocateBytes(len); if (alloc == null) { // The allocation was too large, allocator decided // not to do anything with it. return kv; } assert alloc != null && alloc.getData() != null; System.arraycopy(kv.getBuffer(), kv.getOffset(), alloc.getData(), alloc.getOffset(), len); KeyValue newKv = new KeyValue(alloc.getData(), alloc.getOffset(), len); newKv.setMemstoreTS(kv.getMemstoreTS()); return newKv; }
@Override public Chunk getNewExternalChunk(ChunkCreator.ChunkType chunkType) { MemStoreLAB mslab = this.mslabs.get(0); return mslab.getNewExternalChunk(chunkType); }
this.allocator = new MemStoreLAB(conf);
/** * The process of merging assumes all cells are allocated on mslab. * There is a rare case in which the first immutable segment, * participating in a merge, is a CSLM. * Since the CSLM hasn't been flattened yet, and there is no point in flattening it (since it is * going to be merged), its big cells (for whom size > maxAlloc) must be copied into mslab. * This method copies the passed cell into the first mslab in the mslabs list, * returning either a new cell instance over the copied data, * or null when this cell cannt be copied. */ @Override public Cell forceCopyOfBigCellInto(Cell cell) { MemStoreLAB mslab = this.mslabs.get(0); return mslab.forceCopyOfBigCellInto(cell); }
@Test public void testLABLargeAllocation() { MemStoreLAB mslab = new MemStoreLABImpl(); KeyValue kv = new KeyValue(rk, cf, q, new byte[2 * 1024 * 1024]); Cell newCell = mslab.copyCellInto(kv); assertNull("2MB allocation shouldn't be satisfied by LAB.", newCell); }
/** * Closing a segment before it is being discarded */ public void close() { if (this.memStoreLAB != null) { this.memStoreLAB.close(); } // do not set MSLab to null as scanners may still be reading the data here and need to decrease // the counter when they finish }
/**------------------------------------------------------------------------ * C-tor to be used when new CellChunkImmutableSegment is built as a result of compaction/merge * of a list of older ImmutableSegments. * The given iterator returns the Cells that "survived" the compaction. */ protected CellChunkImmutableSegment(CellComparator comparator, MemStoreSegmentsIterator iterator, MemStoreLAB memStoreLAB, int numOfCells, MemStoreCompactionStrategy.Action action) { super(null, comparator, memStoreLAB); // initialize the CellSet with NULL long indexOverhead = DEEP_OVERHEAD_CCM; // memStoreLAB cannot be null in this class boolean onHeap = getMemStoreLAB().isOnHeap(); // initiate the heapSize with the size of the segment metadata if (onHeap) { incMemStoreSize(0, indexOverhead, 0, 0); } else { incMemStoreSize(0, 0, indexOverhead, 0); } // build the new CellSet based on CellArrayMap and update the CellSet of the new Segment initializeCellSet(numOfCells, iterator, action); }
private Cell maybeCloneWithAllocator(Cell cell) { if (allocator == null) { return cell; } int len = KeyValueUtil.length(cell); ByteRange alloc = allocator.allocateBytes(len); if (alloc == null) { // The allocation was too large, allocator decided // not to do anything with it. return cell; } assert alloc.getBytes() != null; KeyValueUtil.appendToByteArray(cell, alloc.getBytes(), alloc.getOffset()); KeyValue newKv = new KeyValue(alloc.getBytes(), alloc.getOffset(), len); newKv.setSequenceId(cell.getSequenceId()); return newKv; }
@Override public Chunk getNewExternalChunk(int size) { MemStoreLAB mslab = this.mslabs.get(0); return mslab.getNewExternalChunk(size); }
/** * Constructor. * @param c Comparator */ public MemStore(final Configuration conf, final KeyValue.KVComparator c) { this.conf = conf; this.comparator = c; this.comparatorIgnoreTimestamp = this.comparator.getComparatorIgnoringTimestamps(); this.comparatorIgnoreType = this.comparator.getComparatorIgnoringType(); this.kvset = new KeyValueSkipListSet(c); this.snapshot = new KeyValueSkipListSet(c); timeRangeTracker = new TimeRangeTracker(); snapshotTimeRangeTracker = new TimeRangeTracker(); this.size = new AtomicLong(DEEP_OVERHEAD); if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) { this.allocator = new MemStoreLAB(conf); } else { this.allocator = null; } }
KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]); int size = kv.getSerializedSize(); ByteBufferKeyValue newKv = (ByteBufferKeyValue) mslab.copyCellInto(kv); if (newKv.getBuffer() != lastBuffer) { expectedOff = 4; mslab.close(); int chunkCount = chunkCreator.getPoolSize(); assertTrue(chunkCount > 0); mslab.copyCellInto(kv); assertEquals(chunkCount - 1, chunkCreator.getPoolSize());
KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]); int size = kv.getSerializedSize(); ByteBufferKeyValue newKv = (ByteBufferKeyValue) mslab.copyCellInto(kv); if (newKv.getBuffer() != lastBuffer) {
/** * The passed snapshot was successfully persisted; it can be let go. * @param id Id of the snapshot to clean out. * @throws UnexpectedStateException * @see #snapshot() */ @Override public void clearSnapshot(long id) throws UnexpectedStateException { MemStoreLAB tmpAllocator = null; if (this.snapshotId != id) { throw new UnexpectedStateException("Current snapshot id is " + this.snapshotId + ",passed " + id); } // OK. Passed in snapshot is same as current snapshot. If not-empty, // create a new snapshot and let the old one go. if (!this.snapshot.isEmpty()) { this.snapshot = new CellSkipListSet(this.comparator); this.snapshotTimeRangeTracker = new TimeRangeTracker(); } this.snapshotSize = 0; this.snapshotId = -1; if (this.snapshotAllocator != null) { tmpAllocator = this.snapshotAllocator; this.snapshotAllocator = null; } if (tmpAllocator != null) { tmpAllocator.close(); } }