@Override public long getDataSize() { return this.mss.getDataSize(); }
public long getDataSize() { return memStoreSize.getDataSize(); }
void throwException(String title, String regionName) { StringBuilder buf = new StringBuilder(); buf.append(title + ", "); buf.append(getRegionInfo().toString()); buf.append(getRegionInfo().isMetaRegion() ? " meta region " : " "); buf.append("stores: "); for (HStore s : stores.values()) { buf.append(s.getColumnFamilyDescriptor().getNameAsString()); buf.append(" size: "); buf.append(s.getMemStoreSize().getDataSize()); buf.append(" "); } buf.append("end-of-stores"); buf.append(", memstore size "); buf.append(getMemStoreDataSize()); if (getRegionInfo().getRegionNameAsString().startsWith(regionName)) { throw new RuntimeException(buf.toString()); } }
/** * If all stores ended up dropping their snapshots, we can safely drop the prepareFlushResult */ private void dropPrepareFlushIfPossible() { if (writestate.flushing) { boolean canDrop = true; if (prepareFlushResult.storeFlushCtxs != null) { for (Entry<byte[], StoreFlushContext> entry : prepareFlushResult.storeFlushCtxs .entrySet()) { HStore store = getStore(entry.getKey()); if (store == null) { continue; } if (store.getSnapshotSize().getDataSize() > 0) { canDrop = false; break; } } } // this means that all the stores in the region has finished flushing, but the WAL marker // may not have been written or we did not receive it yet. if (canDrop) { writestate.flushing = false; this.prepareFlushResult = null; } } }
/** * Package private constructor. */ MemStoreSize(MemStoreSize memStoreSize) { this.dataSize = memStoreSize.getDataSize(); this.heapSize = memStoreSize.getHeapSize(); this.offHeapSize = memStoreSize.getOffHeapSize(); this.cellsCount = memStoreSize.getCellsCount(); }
@Override public MemStoreSize getFlushableSize() { MemStoreSize mss = getSnapshotSize(); return mss.getDataSize() > 0? mss: getActive().getMemStoreSize(); }
ThreadSafeMemStoreSizing(MemStoreSize mss) { this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
NonThreadSafeMemStoreSizing(MemStoreSize mss) { this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
/** * Utility method broken out of internalPrepareFlushCache so that method is smaller. */ private void logFatLineOnFlush(Collection<HStore> storesToFlush, long sequenceId) { if (!LOG.isInfoEnabled()) { return; } // Log a fat line detailing what is being flushed. StringBuilder perCfExtras = null; if (!isAllFamilies(storesToFlush)) { perCfExtras = new StringBuilder(); for (HStore store: storesToFlush) { MemStoreSize mss = store.getFlushableSize(); perCfExtras.append("; ").append(store.getColumnFamilyName()); perCfExtras.append("={dataSize=") .append(StringUtils.byteDesc(mss.getDataSize())); perCfExtras.append(", heapSize=") .append(StringUtils.byteDesc(mss.getHeapSize())); perCfExtras.append(", offHeapSize=") .append(StringUtils.byteDesc(mss.getOffHeapSize())); perCfExtras.append("}"); } } MemStoreSize mss = this.memStoreSizing.getMemStoreSize(); LOG.info("Flushing " + + storesToFlush.size() + "/" + stores.size() + " column families," + " dataSize=" + StringUtils.byteDesc(mss.getDataSize()) + " heapSize=" + StringUtils.byteDesc(mss.getHeapSize()) + ((perCfExtras != null && perCfExtras.length() > 0)? perCfExtras.toString(): "") + ((wal != null) ? "" : "; WAL is null, using passed sequenceid=" + sequenceId)); }
void incGlobalMemStoreSize(MemStoreSize mss) { incGlobalMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); }
/** * Increase the size of mem store in this region and the size of global mem * store */ void incMemStoreSize(MemStoreSize mss) { incMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
default long decMemStoreSize(MemStoreSize delta) { return incMemStoreSize(-delta.getDataSize(), -delta.getHeapSize(), -delta.getOffHeapSize(), -delta.getCellsCount()); }
void decrMemStoreSize(MemStoreSize mss) { decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
default long incMemStoreSize(MemStoreSize delta) { return incMemStoreSize(delta.getDataSize(), delta.getHeapSize(), delta.getOffHeapSize(), delta.getCellsCount()); }
protected Segment(CellComparator comparator, List<ImmutableSegment> segments, TimeRangeTracker trt) { long dataSize = 0; long heapSize = 0; long OffHeapSize = 0; int cellsCount = 0; for (Segment segment : segments) { MemStoreSize memStoreSize = segment.getMemStoreSize(); dataSize += memStoreSize.getDataSize(); heapSize += memStoreSize.getHeapSize(); OffHeapSize += memStoreSize.getOffHeapSize(); cellsCount += memStoreSize.getCellsCount(); } this.comparator = comparator; this.updatesLock = new ReentrantReadWriteLock(); // Do we need to be thread safe always? What if ImmutableSegment? // DITTO for the TimeRangeTracker below. this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount); this.timeRangeTracker = trt; }
@Override public MemStoreSize getFlushableSize() { MemStoreSize mss = getSnapshotSize(); if (mss.getDataSize() == 0) { // if snapshot is empty the tail of the pipeline (or everything in the memstore) is flushed if (compositeSnapshot) { MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(pipeline.getPipelineSize()); MutableSegment currActive = getActive(); if(!currActive.isEmpty()) { memStoreSizing.incMemStoreSize(currActive.getMemStoreSize()); } mss = memStoreSizing.getMemStoreSize(); } else { mss = pipeline.getTailSize(); } } return mss.getDataSize() > 0? mss: getActive().getMemStoreSize(); }
private long addRowsByKeysDataSize(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); MemStoreSizing memstoreSizing = new NonThreadSafeMemStoreSizing(); for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); hmc.add(kv, memstoreSizing); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); } MemStoreSize mss = memstoreSizing.getMemStoreSize(); regionServicesForStores.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); return mss.getDataSize(); }
/** * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased * as older keyvalues are deleted from the memstore. * @throws Exception */ @Test public void testUpsertMemstoreSize() throws Exception { Configuration conf = HBaseConfiguration.create(); memstore = new DefaultMemStore(conf, CellComparatorImpl.COMPARATOR); MemStoreSize oldSize = memstore.size(); List<Cell> l = new ArrayList<>(); KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v"); KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v"); kv1.setSequenceId(1); kv2.setSequenceId(1);kv3.setSequenceId(1); l.add(kv1); l.add(kv2); l.add(kv3); this.memstore.upsert(l, 2, null);// readpoint is 2 MemStoreSize newSize = this.memstore.size(); assert (newSize.getDataSize() > oldSize.getDataSize()); //The kv1 should be removed. assert(memstore.getActive().getCellsCount() == 2); KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v"); kv4.setSequenceId(1); l.clear(); l.add(kv4); this.memstore.upsert(l, 3, null); assertEquals(newSize, this.memstore.size()); //The kv2 should be removed. assert(memstore.getActive().getCellsCount() == 2); //this.memstore = null; }
assert (newSize.getDataSize() > oldSize.getDataSize());
@Test public void testPutSameCell() { byte[] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); MemStoreSizing sizeChangeForFirstCell = new NonThreadSafeMemStoreSizing(); this.memstore.add(kv, sizeChangeForFirstCell); MemStoreSizing sizeChangeForSecondCell = new NonThreadSafeMemStoreSizing(); this.memstore.add(kv, sizeChangeForSecondCell); // make sure memstore size increase won't double-count MSLAB chunk size assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getMemStoreSize().getDataSize()); Segment segment = this.memstore.getActive(); MemStoreLAB msLab = segment.getMemStoreLAB(); if (msLab != null) { // make sure memstore size increased even when writing the same cell, if using MSLAB assertEquals(Segment.getCellLength(kv), sizeChangeForSecondCell.getMemStoreSize().getDataSize()); // make sure chunk size increased even when writing the same cell, if using MSLAB if (msLab instanceof MemStoreLABImpl) { // since we add the chunkID at the 0th offset of the chunk and the // chunkid is an int we need to account for those 4 bytes assertEquals(2 * Segment.getCellLength(kv) + Bytes.SIZEOF_INT, ((MemStoreLABImpl) msLab).getCurrentChunk().getNextFreeOffset()); } } else { // make sure no memstore size change w/o MSLAB assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getDataSize()); assertEquals(0, sizeChangeForSecondCell.getMemStoreSize().getHeapSize()); } }