protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed, MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) { long delta = 0; long cellSize = getCellLength(cellToAdd); int cellsCount = succ ? 1 : 0; // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger // than the counted number) if (succ || mslabUsed) { delta = cellSize; } if (sizeAddedPreOperation) { delta -= cellSize; } long heapSize = heapSizeChange(cellToAdd, succ || mslabUsed); long offHeapSize = offHeapSizeChange(cellToAdd, succ || mslabUsed); incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); if (memstoreSizing != null) { memstoreSizing.incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); } getTimeRangeTracker().includeTimestamp(cellToAdd); minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. // When we use ACL CP or Visibility CP which deals with Tags during // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not // parse the byte[] to identify the tags length. if (cellToAdd.getTagsLength() > 0) { tagsPresent = true; } }
@Override public String toString() { String res = "type=" + this.getClass().getSimpleName() + ", "; res += "empty=" + (isEmpty()? "yes": "no") + ", "; res += "cellCount=" + getCellsCount() + ", "; res += "cellSize=" + getDataSize() + ", "; res += "totalHeapSize=" + getHeapSize() + ", "; res += "min timestamp=" + timeRangeTracker.getMin() + ", "; res += "max timestamp=" + timeRangeTracker.getMax(); return res; }
protected Segment(Segment segment) { this.cellSet.set(segment.getCellSet()); this.comparator = segment.getComparator(); this.updatesLock = segment.getUpdatesLock(); this.minSequenceId = segment.getMinSequenceId(); this.memStoreLAB = segment.getMemStoreLAB(); this.memStoreSizing = segment.memStoreSizing; this.tagsPresent = segment.isTagsPresent(); this.timeRangeTracker = segment.getTimeRangeTracker(); }
protected void updateMetaInfo(Cell cellToAdd, boolean succ, MemStoreSizing memstoreSizing) { updateMetaInfo(cellToAdd, succ, (getMemStoreLAB()!=null), memstoreSizing, false); }
protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) { boolean succ = getCellSet().add(cell); updateMetaInfo(cell, succ, mslabUsed, memstoreSizing, sizeAddedPreOperation); }
private static int getSegmentsCellsCount(List<? extends Segment> list) { int res = 0; for (Segment segment : list) { res += segment.getCellsCount(); } return res; }
@Test public void testPutSameCell() { byte[] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); MemStoreSizing sizeChangeForFirstCell = new NonThreadSafeMemStoreSizing(); this.memstore.add(kv, sizeChangeForFirstCell); MemStoreSizing sizeChangeForSecondCell = new NonThreadSafeMemStoreSizing(); this.memstore.add(kv, sizeChangeForSecondCell); // make sure memstore size increase won't double-count MSLAB chunk size assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getMemStoreSize().getDataSize()); Segment segment = this.memstore.getActive(); MemStoreLAB msLab = segment.getMemStoreLAB(); if (msLab != null) { // make sure memstore size increased even when writing the same cell, if using MSLAB assertEquals(Segment.getCellLength(kv), sizeChangeForSecondCell.getMemStoreSize().getDataSize()); // make sure chunk size increased even when writing the same cell, if using MSLAB if (msLab instanceof MemStoreLABImpl) { // since we add the chunkID at the 0th offset of the chunk and the // chunkid is an int we need to account for those 4 bytes assertEquals(2 * Segment.getCellLength(kv) + Bytes.SIZEOF_INT, ((MemStoreLABImpl) msLab).getCurrentChunk().getNextFreeOffset()); } } else { // make sure no memstore size change w/o MSLAB assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getDataSize()); assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getHeapSize()); } }
/** * When a cell's size is too big (bigger than maxAlloc), * copyCellInto does not allocate it on MSLAB. * Since the process of flattening to CellChunkMap assumes that * all cells are allocated on MSLAB, during this process, * the big cells are copied into MSLAB using this method. */ @Override public Cell forceCopyOfBigCellInto(Cell cell) { int size = Segment.getCellLength(cell); size += ChunkCreator.SIZEOF_CHUNK_HEADER; Preconditions.checkArgument(size >= 0, "negative size"); if (size <= dataChunkSize) { // Using copyCellInto for cells which are bigger than the original maxAlloc return copyCellInto(cell, dataChunkSize); } else { Chunk c = getNewExternalChunk(size); int allocOffset = c.alloc(size); return copyToChunkCell(cell, c.getData(), allocOffset, size); } }
protected long offHeapSizeChange(Cell cell, boolean allocated) { long res = 0; if (allocated) { boolean offHeap = false; MemStoreLAB memStoreLAB = getMemStoreLAB(); if(memStoreLAB != null) { offHeap = memStoreLAB.isOffHeap(); } res += indexEntryOffHeapSize(offHeap); if(offHeap) { res += cell.heapSize(); } res = ClassSize.align(res); } return res; }
/** * @return The increase in heap size because of this cell addition. This includes this cell POJO's * heap size itself and additional overhead because of addition on to CSLM. */ protected long heapSizeChange(Cell cell, boolean allocated) { long res = 0; if (allocated) { boolean onHeap = true; MemStoreLAB memStoreLAB = getMemStoreLAB(); if(memStoreLAB != null) { onHeap = memStoreLAB.isOnHeap(); } res += indexEntryOnHeapSize(onHeap); if(onHeap) { res += cell.heapSize(); } res = ClassSize.align(res); } return res; }
/** * Dumps all cells of the segment into the given log */ void dump(Logger log) { for (Cell cell: getCellSet()) { log.debug(Objects.toString(cell)); } }
assertEquals(numberOfCell, memstore.getSegments().stream().mapToInt(Segment::getCellsCount).sum()); assertEquals(minTs, memstore.getSegments().stream().mapToLong( m -> m.getTimeRangeTracker().getMin()).min().getAsLong()); assertEquals(currentTs, memstore.getSegments().stream().mapToLong( m -> m.getTimeRangeTracker().getMax()).max().getAsLong()); assertEquals(numberOfCell, memstore.getSegments().stream().mapToInt(Segment::getCellsCount).sum()); assertEquals(minTs, memstore.getSegments().stream().mapToLong( m -> m.getTimeRangeTracker().getMin()).min().getAsLong()); assertEquals(currentTs, memstore.getSegments().stream().mapToLong( m -> m.getTimeRangeTracker().getMax()).max().getAsLong()); m -> m.getTimeRangeTracker().getMin()).min().getAsLong()); assertEquals(currentTs, memstore.getSegments().stream().mapToLong( m -> m.getTimeRangeTracker().getMax()).max().getAsLong());
do { Cell firstKeyOnRow = PrivateCellUtil.createFirstOnRow(key); SortedSet<Cell> cellHead = segment.headSet(firstKeyOnRow); Cell lastCellBeforeRow = cellHead.isEmpty() ? null : cellHead.last(); if (lastCellBeforeRow == null) { this.stopSkippingKVsIfNextRow = false; if (peek() == null || segment.getComparator().compareRows(peek(), firstKeyOnPreviousRow) > 0) { keepSeeking = true; key = firstKeyOnPreviousRow;
public int compareRows(Cell left, Cell right) { return getComparator().compareRows(left, right); }
private void swapSuffix(List<? extends Segment> suffix, ImmutableSegment segment, boolean closeSegmentsInSuffix) { pipeline.removeAll(suffix); if(segment != null) pipeline.addLast(segment); // During index merge we won't be closing the segments undergoing the merge. Segment#close() // will release the MSLAB chunks to pool. But in case of index merge there wont be any data copy // from old MSLABs. So the new cells in new segment also refers to same chunks. In case of data // compaction, we would have copied the cells data from old MSLAB chunks into a new chunk // created for the result segment. So we can release the chunks associated with the compacted // segments. if (closeSegmentsInSuffix) { for (Segment itemInSuffix : suffix) { itemInSuffix.close(); } } }
/** * Private internal method for iterating over the segment, * skipping the cells with irrelevant MVCC */ protected void updateCurrent() { Cell next = null; try { while (iter.hasNext()) { next = iter.next(); if (next.getSequenceId() <= this.readPoint) { current = next; return;// skip irrelevant versions } // for backwardSeek() stay in the boundaries of a single row if (stopSkippingKVsIfNextRow && segment.compareRows(next, stopSkippingKVsRow) > 0) { current = null; return; } } // end of while current = null; // nothing found } finally { if (next != null) { // in all cases, remember the last KV we iterated to, needed for reseek() last = next; } } }
/** * Private internal method that returns the higher of the two key values, or null * if they are both null */ private Cell getHighest(Cell first, Cell second) { if (first == null && second == null) { return null; } if (first != null && second != null) { int compare = segment.compare(first, second); return (compare > 0 ? first : second); } return (first != null ? first : second); } }
/** * Close the KeyValue scanner. */ @Override public void close() { if (closed) { return; } getSegment().decScannerCount(); closed = true; }
@Override public void updateStats(Segment replacement) { if(compacted) { if (replacement.getCellsCount() / numCellsInVersionedList < 1.0 - compactionThreshold) { // compaction was a good decision - increase probability compactionProbability *= ADAPTIVE_PROBABILITY_FACTOR; if(compactionProbability > 1.0) { compactionProbability = 1.0; } } else { // compaction was NOT a good decision - decrease probability compactionProbability /= ADAPTIVE_PROBABILITY_FACTOR; } } }
this.memstore.add(kv, sizeChangeForSecondCell); assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getMemStoreSize().getDataSize()); Segment segment = this.memstore.getActive(); MemStoreLAB msLab = segment.getMemStoreLAB(); if (msLab != null) { if (msLab.isOnHeap()) { assertEquals(Segment.getCellLength(kv), sizeChangeForSecondCell.getMemStoreSize().getDataSize()); assertEquals(2 * Segment.getCellLength(kv) + Bytes.SIZEOF_INT, ((MemStoreLABImpl) msLab).getCurrentChunk().getNextFreeOffset());