/** * When a cell's size is too big (bigger than maxAlloc), * copyCellInto does not allocate it on MSLAB. * Since the process of flattening to CellChunkMap assumes that * all cells are allocated on MSLAB, during this process, * the big cells are copied into MSLAB using this method. */ @Override public Cell forceCopyOfBigCellInto(Cell cell) { int size = Segment.getCellLength(cell); size += ChunkCreator.SIZEOF_CHUNK_HEADER; Preconditions.checkArgument(size >= 0, "negative size"); if (size <= dataChunkSize) { // Using copyCellInto for cells which are bigger than the original maxAlloc return copyCellInto(cell, dataChunkSize); } else { Chunk c = getNewExternalChunk(size); int allocOffset = c.alloc(size); return copyToChunkCell(cell, c.getData(), allocOffset, size); } }
protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed, MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) { long delta = 0; long cellSize = getCellLength(cellToAdd); int cellsCount = succ ? 1 : 0; // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger // than the counted number) if (succ || mslabUsed) { delta = cellSize; } if (sizeAddedPreOperation) { delta -= cellSize; } long heapSize = heapSizeChange(cellToAdd, succ || mslabUsed); long offHeapSize = offHeapSizeChange(cellToAdd, succ || mslabUsed); incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); if (memstoreSizing != null) { memstoreSizing.incMemStoreSize(delta, heapSize, offHeapSize, cellsCount); } getTimeRangeTracker().includeTimestamp(cellToAdd); minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. // When we use ACL CP or Visibility CP which deals with Tags during // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not // parse the byte[] to identify the tags length. if (cellToAdd.getTagsLength() > 0) { tagsPresent = true; } }
int size = Segment.getCellLength(cell); Preconditions.checkArgument(size >= 0, "negative size");
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys, byte[] val) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += Segment.getCellLength(kv); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0, cellsCount); return totalLen; }
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += Segment.getCellLength(kv); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0, hmc.getActive().getCellsCount() - cellsCount); return totalLen; }
@Test public void testPutSameCell() { byte[] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); MemStoreSizing sizeChangeForFirstCell = new NonThreadSafeMemStoreSizing(); this.memstore.add(kv, sizeChangeForFirstCell); MemStoreSizing sizeChangeForSecondCell = new NonThreadSafeMemStoreSizing(); this.memstore.add(kv, sizeChangeForSecondCell); // make sure memstore size increase won't double-count MSLAB chunk size assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getMemStoreSize().getDataSize()); Segment segment = this.memstore.getActive(); MemStoreLAB msLab = segment.getMemStoreLAB(); if (msLab != null) { // make sure memstore size increased even when writing the same cell, if using MSLAB assertEquals(Segment.getCellLength(kv), sizeChangeForSecondCell.getMemStoreSize().getDataSize()); // make sure chunk size increased even when writing the same cell, if using MSLAB if (msLab instanceof MemStoreLABImpl) { // since we add the chunkID at the 0th offset of the chunk and the // chunkid is an int we need to account for those 4 bytes assertEquals(2 * Segment.getCellLength(kv) + Bytes.SIZEOF_INT, ((MemStoreLABImpl) msLab).getCurrentChunk().getNextFreeOffset()); } } else { // make sure no memstore size change w/o MSLAB assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getDataSize()); assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getHeapSize()); } }
this.memstore.add(kv, sizeChangeForSecondCell); assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getMemStoreSize().getDataSize()); Segment segment = this.memstore.getActive(); MemStoreLAB msLab = segment.getMemStoreLAB(); assertEquals(Segment.getCellLength(kv), sizeChangeForSecondCell.getMemStoreSize().getDataSize()); assertEquals(2 * Segment.getCellLength(kv) + Bytes.SIZEOF_INT, ((MemStoreLABImpl) msLab).getCurrentChunk().getNextFreeOffset());