/** * @return The new dataSize ONLY as a convenience */ default long decMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, int cellsCountDelta) { return incMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta, -cellsCountDelta); }
@Override public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) { return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead, cellsCount); }
/**------------------------------------------------------------------------ * Copy C-tor to be used when new CSLMImmutableSegment is being built from a Mutable one. * This C-tor should be used when active MutableSegment is pushed into the compaction * pipeline and becomes an ImmutableSegment. */ protected CSLMImmutableSegment(Segment segment, MemStoreSizing memstoreSizing) { super(segment); // update the segment metadata heap size long indexOverhead = -MutableSegment.DEEP_OVERHEAD + DEEP_OVERHEAD_CSLM; incMemStoreSize(0, indexOverhead, 0, 0); // CSLM is always on-heap if (memstoreSizing != null) { memstoreSizing.incMemStoreSize(0, indexOverhead, 0, 0); } }
protected MutableSegment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, MemStoreSizing memstoreSizing) { super(cellSet, comparator, memStoreLAB, TimeRangeTracker.create(TimeRangeTracker.Type.SYNC)); incMemStoreSize(0, DEEP_OVERHEAD, 0, 0); // update the mutable segment metadata if (memstoreSizing != null) { memstoreSizing.incMemStoreSize(0, DEEP_OVERHEAD, 0, 0); } }
void incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, int cellsCountDelta) { if (this.rsAccounting != null) { rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta); } long dataSize = this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta, cellsCountDelta); checkNegativeMemStoreDataSize(dataSize, dataSizeDelta); }
public MemStoreSize getPipelineSize() { MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); LinkedList<? extends Segment> localCopy = readOnlyCopy; for (Segment segment : localCopy) { memStoreSizing.incMemStoreSize(segment.getMemStoreSize()); } return memStoreSizing.getMemStoreSize(); }
/** * Be careful, this method will drop all data in the memstore of this region. * Currently, this method is used to drop memstore to prevent memory leak * when replaying recovered.edits while opening region. */ public MemStoreSize dropMemStoreContents() throws IOException { MemStoreSizing totalFreedSize = new NonThreadSafeMemStoreSizing(); this.updatesLock.writeLock().lock(); try { for (HStore s : stores.values()) { MemStoreSize memStoreSize = doDropStoreMemStoreContentsForSeqId(s, HConstants.NO_SEQNUM); LOG.info("Drop memstore for Store " + s.getColumnFamilyName() + " in region " + this.getRegionInfo().getRegionNameAsString() + " , dropped memstoresize: [" + memStoreSize + " }"); totalFreedSize.incMemStoreSize(memStoreSize); } return totalFreedSize.getMemStoreSize(); } finally { this.updatesLock.writeLock().unlock(); } }
/** * @return Total memory occupied by this MemStore. This won't include any size occupied by the * snapshot. We assume the snapshot will get cleared soon. This is not thread safe and * the memstore may be changed while computing its size. It is the responsibility of the * caller to make sure this doesn't happen. */ @Override public MemStoreSize size() { MemStoreSizing memstoreSizing = new NonThreadSafeMemStoreSizing(); memstoreSizing.incMemStoreSize(getActive().getMemStoreSize()); for (Segment item : pipeline.getSegments()) { memstoreSizing.incMemStoreSize(item.getMemStoreSize()); } return memstoreSizing.getMemStoreSize(); }
totalFreedSize.incMemStoreSize(doDropStoreMemStoreContentsForSeqId(s, currentSeqId)); totalFreedSize.incMemStoreSize(doDropStoreMemStoreContentsForSeqId(store, currentSeqId));
default long incMemStoreSize(MemStoreSize delta) { return incMemStoreSize(delta.getDataSize(), delta.getHeapSize(), delta.getOffHeapSize(), delta.getCellsCount()); }
default long decMemStoreSize(MemStoreSize delta) { return incMemStoreSize(-delta.getDataSize(), -delta.getHeapSize(), -delta.getOffHeapSize(), -delta.getCellsCount()); }
/**------------------------------------------------------------------------ * C-tor to be used when new CellChunkImmutableSegment is built as a result of flattening * of CSLMImmutableSegment * The given iterator returns the Cells that "survived" the compaction. */ protected CellArrayImmutableSegment(CSLMImmutableSegment segment, MemStoreSizing mss, MemStoreCompactionStrategy.Action action) { super(segment); // initiailize the upper class long indexOverhead = DEEP_OVERHEAD_CAM - CSLMImmutableSegment.DEEP_OVERHEAD_CSLM; incMemStoreSize(0, indexOverhead, 0, 0); // CAM is always on-heap mss.incMemStoreSize(0, indexOverhead, 0, 0); int numOfCells = segment.getCellsCount(); // build the new CellSet based on CellChunkMap and update the CellSet of this Segment reinitializeCellSet(numOfCells, segment.getScanner(Long.MAX_VALUE), segment.getCellSet(), action); // arrange the meta-data size, decrease all meta-data sizes related to SkipList; // add sizes of CellArrayMap entry (reinitializeCellSet doesn't take the care for the sizes) long newSegmentSizeDelta = numOfCells * (indexEntrySize() - ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); incMemStoreSize(0, newSegmentSizeDelta, 0, 0); mss.incMemStoreSize(0, newSegmentSizeDelta, 0, 0); }
memstoreSizing.incMemStoreSize(0, indexOverhead, 0, 0); } else { incMemStoreSize(0, -CSLMImmutableSegment.DEEP_OVERHEAD_CSLM, DEEP_OVERHEAD_CCM, 0); memstoreSizing.incMemStoreSize(0, -CSLMImmutableSegment.DEEP_OVERHEAD_CSLM, DEEP_OVERHEAD_CCM, 0); if(onHeap) { incMemStoreSize(0, newSegmentSizeDelta, 0, 0); memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0, 0); } else { incMemStoreSize(0, 0, newSegmentSizeDelta, 0); memstoreSizing.incMemStoreSize(0, 0, newSegmentSizeDelta, 0);
@VisibleForTesting protected boolean shouldFlushInMemory(MutableSegment currActive, Cell cellToAdd, MemStoreSizing memstoreSizing) { long cellSize = currActive.getCellLength(cellToAdd); long segmentDataSize = currActive.getDataSize(); while (segmentDataSize + cellSize < inmemoryFlushSize || inWalReplay) { // when replaying edits from WAL there is no need in in-memory flush regardless the size // otherwise size below flush threshold try to update atomically if (currActive.compareAndSetDataSize(segmentDataSize, segmentDataSize + cellSize)) { if (memstoreSizing != null) { memstoreSizing.incMemStoreSize(cellSize, 0, 0, 0); } // enough space for cell - no need to flush return false; } segmentDataSize = currActive.getDataSize(); } // size above flush threshold return true; }
protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed, MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) { long delta = 0; long cellSize = getCellLength(cellToAdd); int cellsCount = succ ? 1 : 0; // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger // than the counted number) if (succ || mslabUsed) { delta = cellSize; } if (sizeAddedPreOperation) { delta -= cellSize; } long heapSize = heapSizeChange(cellToAdd, succ || mslabUsed); long offHeapSize = offHeapSizeChange(cellToAdd, succ || mslabUsed); incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); if (memstoreSizing != null) { memstoreSizing.incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); } getTimeRangeTracker().includeTimestamp(cellToAdd); minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. // When we use ACL CP or Visibility CP which deals with Tags during // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not // parse the byte[] to identify the tags length. if (cellToAdd.getTagsLength() > 0) { tagsPresent = true; } }
private Cell copyCellIntoMSLAB(Cell cell, MemStoreSizing memstoreSizing) { // Take care for a special case when a cell is copied from on-heap to (probably off-heap) MSLAB. // The cell allocated as an on-heap JVM object (byte array) occupies slightly different // amount of memory, than when the cell serialized and allocated on the MSLAB. // Here, we update the heap size of the new segment only for the difference between object and // serialized size. This is a decrease of the size as serialized cell is a bit smaller. // The actual size of the cell is not added yet, and will be added (only in compaction) // in initializeCellSet#updateMetaInfo(). long oldHeapSize = heapSizeChange(cell, true); long oldOffHeapSize = offHeapSizeChange(cell, true); long oldCellSize = getCellLength(cell); cell = maybeCloneWithAllocator(cell, true); long newHeapSize = heapSizeChange(cell, true); long newOffHeapSize = offHeapSizeChange(cell, true); long newCellSize = getCellLength(cell); long heapOverhead = newHeapSize - oldHeapSize; long offHeapOverhead = newOffHeapSize - oldOffHeapSize; incMemStoreSize(newCellSize - oldCellSize, heapOverhead, offHeapOverhead, 0); if(memstoreSizing != null) { memstoreSizing.incMemStoreSize(newCellSize - oldCellSize, heapOverhead, offHeapOverhead, 0); } return cell; } }
totalSizeOfFlushableStores.incMemStoreSize(snapshotSize); storeFlushableSize.put(name, snapshotSize); });
@Override public MemStoreSize getFlushableSize() { MemStoreSize mss = getSnapshotSize(); if (mss.getDataSize() == 0) { // if snapshot is empty the tail of the pipeline (or everything in the memstore) is flushed if (compositeSnapshot) { MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(pipeline.getPipelineSize()); MutableSegment currActive = getActive(); if(!currActive.isEmpty()) { memStoreSizing.incMemStoreSize(currActive.getMemStoreSize()); } mss = memStoreSizing.getMemStoreSize(); } else { mss = pipeline.getTailSize(); } } return mss.getDataSize() > 0? mss: getActive().getMemStoreSize(); }
store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize); kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0); mss = store.memstore.getFlushableSize(); assertEquals(kvSize.getMemStoreSize(), mss); kvSize.incMemStoreSize(0, CSLMImmutableSegment.DEEP_OVERHEAD_CSLM - MutableSegment.DEEP_OVERHEAD, 0, 0); mss = store.memstore.getFlushableSize(); MemStoreSizing kvSize2 = new NonThreadSafeMemStoreSizing(); store.add(new KeyValue(row, family, qf2, 2, (byte[]) null), kvSize2); kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0);
store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize); kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0); mss = store.memstore.getFlushableSize(); assertEquals(kvSize.getMemStoreSize(), mss); kvSize.incMemStoreSize(0, CSLMImmutableSegment.DEEP_OVERHEAD_CSLM-MutableSegment.DEEP_OVERHEAD, 0); mss = store.memstore.getFlushableSize(); MemStoreSizing kvSize2 = new NonThreadSafeMemStoreSizing(); store.add(new KeyValue(row, family, qf2, 2, (byte[])null), kvSize2); kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0);