public MemStoreSize getPipelineSize() { MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); LinkedList<? extends Segment> localCopy = readOnlyCopy; for (Segment segment : localCopy) { memStoreSizing.incMemStoreSize(segment.getMemStoreSize()); } return memStoreSizing.getMemStoreSize(); }
/** * Be careful, this method will drop all data in the memstore of this region. * Currently, this method is used to drop memstore to prevent memory leak * when replaying recovered.edits while opening region. */ public MemStoreSize dropMemStoreContents() throws IOException { MemStoreSizing totalFreedSize = new NonThreadSafeMemStoreSizing(); this.updatesLock.writeLock().lock(); try { for (HStore s : stores.values()) { MemStoreSize memStoreSize = doDropStoreMemStoreContentsForSeqId(s, HConstants.NO_SEQNUM); LOG.info("Drop memstore for Store " + s.getColumnFamilyName() + " in region " + this.getRegionInfo().getRegionNameAsString() + " , dropped memstoresize: [" + memStoreSize + " }"); totalFreedSize.incMemStoreSize(memStoreSize); } return totalFreedSize.getMemStoreSize(); } finally { this.updatesLock.writeLock().unlock(); } }
MemStoreSizing totalFreedSize = new NonThreadSafeMemStoreSizing(); this.updatesLock.writeLock().lock(); try {
byte [] qf = Bytes.toBytes("umn"); byte [] empty = new byte[0]; MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); for (int i = 0; i < count; i++) { memStoreSizing = new NonThreadSafeMemStoreSizing(); for (int i = 0; i < count; i++) { memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]), memStoreSizing);
public boolean pushHead(MutableSegment segment) { // Record the ImmutableSegment' heap overhead when initialing MemStoreSizing memstoreAccounting = new NonThreadSafeMemStoreSizing(); ImmutableSegment immutableSegment = SegmentFactory.instance(). createImmutableSegment(segment, memstoreAccounting); if (region != null) { region.addMemStoreSize(memstoreAccounting.getDataSize(), memstoreAccounting.getHeapSize(), memstoreAccounting.getOffHeapSize(), memstoreAccounting.getCellsCount()); } synchronized (pipeline){ boolean res = addFirst(immutableSegment); readOnlyCopy = new LinkedList<>(pipeline); return res; } }
protected void resetActive() { // Record the MutableSegment' heap overhead when initialing MemStoreSizing memstoreAccounting = new NonThreadSafeMemStoreSizing(); // Reset heap to not include any keys active = SegmentFactory.instance().createMutableSegment(conf, comparator, memstoreAccounting); // regionServices can be null when testing if (regionServices != null) { regionServices.addMemStoreSize(memstoreAccounting.getDataSize(), memstoreAccounting.getHeapSize(), memstoreAccounting.getOffHeapSize(), memstoreAccounting.getCellsCount()); } timeOfOldestEdit = Long.MAX_VALUE; }
/** * @return Total memory occupied by this MemStore. This won't include any size occupied by the * snapshot. We assume the snapshot will get cleared soon. This is not thread safe and * the memstore may be changed while computing its size. It is the responsibility of the * caller to make sure this doesn't happen. */ @Override public MemStoreSize size() { MemStoreSizing memstoreSizing = new NonThreadSafeMemStoreSizing(); memstoreSizing.incMemStoreSize(getActive().getMemStoreSize()); for (Segment item : pipeline.getSegments()) { memstoreSizing.incMemStoreSize(item.getMemStoreSize()); } return memstoreSizing.getMemStoreSize(); }
MemStoreSizing newMemstoreAccounting = new NonThreadSafeMemStoreSizing(); ImmutableSegment newS = SegmentFactory.instance().createImmutableSegmentByFlattening( (CSLMImmutableSegment)s,idxType,newMemstoreAccounting,action);
protected void writeMiniBatchOperationsToMemStore( final MiniBatchOperationInProgress<Mutation> miniBatchOp, final long writeNumber) throws IOException { MemStoreSizing memStoreAccounting = new NonThreadSafeMemStoreSizing(); visitBatchOperations(true, miniBatchOp.getLastIndexExclusive(), (int index) -> { // We need to update the sequence id for following reasons. // 1) If the op is in replay mode, FSWALEntry#stampRegionSequenceId won't stamp sequence id. // 2) If no WAL, FSWALEntry won't be used // we use durability of the original mutation for the mutation passed by CP. if (isInReplay() || getMutation(index).getDurability() == Durability.SKIP_WAL) { region.updateSequenceId(familyCellMaps[index].values(), writeNumber); } applyFamilyMapToMemStore(familyCellMaps[index], memStoreAccounting); return true; }); // update memStore size region.incMemStoreSize(memStoreAccounting.getDataSize(), memStoreAccounting.getHeapSize(), memStoreAccounting.getOffHeapSize(), memStoreAccounting.getCellsCount()); }
private long addRowsByKeysDataSize(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); MemStoreSizing memstoreSizing = new NonThreadSafeMemStoreSizing(); for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); hmc.add(kv, memstoreSizing); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); } MemStoreSize mss = memstoreSizing.getMemStoreSize(); regionServicesForStores.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); return mss.getDataSize(); }
.setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build()); byte[] value = Bytes.toBytes("value"); MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); long ts = EnvironmentEdgeManager.currentTime(); long seqId = 100;
/** * Creates a snapshot of the current memstore. * Snapshot must be cleared by call to {@link #clearSnapshot(long)} */ @Override public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); if (!getActive().isEmpty()) { // Record the ImmutableSegment' heap overhead when initialing MemStoreSizing memstoreAccounting = new NonThreadSafeMemStoreSizing(); ImmutableSegment immutableSegment = SegmentFactory.instance(). createImmutableSegment(getActive(), memstoreAccounting); // regionServices can be null when testing if (regionServices != null) { regionServices.addMemStoreSize(memstoreAccounting.getDataSize(), memstoreAccounting.getHeapSize(), memstoreAccounting.getOffHeapSize(), memstoreAccounting.getCellsCount()); } this.snapshot = immutableSegment; resetActive(); } } return new MemStoreSnapshot(this.snapshotId, this.snapshot); }
@Override public MemStoreSize getFlushableSize() { MemStoreSize mss = getSnapshotSize(); if (mss.getDataSize() == 0) { // if snapshot is empty the tail of the pipeline (or everything in the memstore) is flushed if (compositeSnapshot) { MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(pipeline.getPipelineSize()); MutableSegment currActive = getActive(); if(!currActive.isEmpty()) { memStoreSizing.incMemStoreSize(currActive.getMemStoreSize()); } mss = memStoreSizing.getMemStoreSize(); } else { mss = pipeline.getTailSize(); } } return mss.getDataSize() > 0? mss: getActive().getMemStoreSize(); }
List<Cell> results = returnResults? new ArrayList<>(mutation.size()): null; RowLock rowLock = null; MemStoreSizing memstoreAccounting = new NonThreadSafeMemStoreSizing(); try { rowLock = getRowLockInternal(mutation.getRow(), false, null);
MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); long ts = System.currentTimeMillis(); long seqID = 1L;
.setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build()); byte[] value = Bytes.toBytes("thisisavarylargevalue"); MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); long ts = EnvironmentEdgeManager.currentTime(); long seqId = 100;
byte[] oldValue = Bytes.toBytes("oldValue"); byte[] currentValue = Bytes.toBytes("currentValue"); MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); long ts = EnvironmentEdgeManager.currentTime(); long seqId = 100;
assertEquals(0, mss.getDataSize()); LOG.info("Adding some data"); MemStoreSizing kvSize = new NonThreadSafeMemStoreSizing(); store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize); mss = store.memstore.getFlushableSize(); assertEquals(kvSize.getMemStoreSize(), mss); MemStoreSizing kvSize2 = new NonThreadSafeMemStoreSizing(); store.add(new KeyValue(row, family, qf2, 2, (byte[]) null), kvSize2); kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0);
@Test public void testPutSameCell() { byte[] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); MemStoreSizing sizeChangeForFirstCell = new NonThreadSafeMemStoreSizing(); this.memstore.add(kv, sizeChangeForFirstCell); MemStoreSizing sizeChangeForSecondCell = new NonThreadSafeMemStoreSizing(); this.memstore.add(kv, sizeChangeForSecondCell); // make sure memstore size increase won't double-count MSLAB chunk size assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getMemStoreSize().getDataSize()); Segment segment = this.memstore.getActive(); MemStoreLAB msLab = segment.getMemStoreLAB(); if (msLab != null) { // make sure memstore size increased even when writing the same cell, if using MSLAB assertEquals(Segment.getCellLength(kv), sizeChangeForSecondCell.getMemStoreSize().getDataSize()); // make sure chunk size increased even when writing the same cell, if using MSLAB if (msLab instanceof MemStoreLABImpl) { // since we add the chunkID at the 0th offset of the chunk and the // chunkid is an int we need to account for those 4 bytes assertEquals(2 * Segment.getCellLength(kv) + Bytes.SIZEOF_INT, ((MemStoreLABImpl) msLab).getCurrentChunk().getNextFreeOffset()); } } else { // make sure no memstore size change w/o MSLAB assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getDataSize()); assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getHeapSize()); } }
byte[] value1 = Bytes.toBytes("value1"); byte[] value2 = Bytes.toBytes("value2"); MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); long ts = EnvironmentEdgeManager.currentTime(); long seqId = 100;