@Override public long getOffHeapSize() { return this.mss.getOffHeapSize(); }
private boolean isFlushSize(MemStoreSize size) { return size.getHeapSize() + size.getOffHeapSize() > getMemStoreFlushSize(); }
/** * Package private constructor. */ MemStoreSize(MemStoreSize memStoreSize) { this.dataSize = memStoreSize.getDataSize(); this.heapSize = memStoreSize.getHeapSize(); this.offHeapSize = memStoreSize.getOffHeapSize(); this.cellsCount = memStoreSize.getCellsCount(); }
void incGlobalMemStoreSize(MemStoreSize mss) { incGlobalMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); }
ThreadSafeMemStoreSizing(MemStoreSize mss) { this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
NonThreadSafeMemStoreSizing(MemStoreSize mss) { this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
/** * Utility method broken out of internalPrepareFlushCache so that method is smaller. */ private void logFatLineOnFlush(Collection<HStore> storesToFlush, long sequenceId) { if (!LOG.isInfoEnabled()) { return; } // Log a fat line detailing what is being flushed. StringBuilder perCfExtras = null; if (!isAllFamilies(storesToFlush)) { perCfExtras = new StringBuilder(); for (HStore store: storesToFlush) { MemStoreSize mss = store.getFlushableSize(); perCfExtras.append("; ").append(store.getColumnFamilyName()); perCfExtras.append("={dataSize=") .append(StringUtils.byteDesc(mss.getDataSize())); perCfExtras.append(", heapSize=") .append(StringUtils.byteDesc(mss.getHeapSize())); perCfExtras.append(", offHeapSize=") .append(StringUtils.byteDesc(mss.getOffHeapSize())); perCfExtras.append("}"); } } MemStoreSize mss = this.memStoreSizing.getMemStoreSize(); LOG.info("Flushing " + + storesToFlush.size() + "/" + stores.size() + " column families," + " dataSize=" + StringUtils.byteDesc(mss.getDataSize()) + " heapSize=" + StringUtils.byteDesc(mss.getHeapSize()) + ((perCfExtras != null && perCfExtras.length() > 0)? perCfExtras.toString(): "") + ((wal != null) ? "" : "; WAL is null, using passed sequenceid=" + sequenceId)); }
/** * Increase the size of mem store in this region and the size of global mem * store */ void incMemStoreSize(MemStoreSize mss) { incMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
protected boolean shouldFlush(HStore store) { if (store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize() > this.flushSizeLowerBound) { LOG.debug("Flush {} of {}; " + "heap memstoreSize={} +" + "off heap memstoreSize={} > memstore lowerBound={}" , store.getColumnFamilyName() , region.getRegionInfo().getEncodedName() , store.getMemStoreSize().getHeapSize() , store.getMemStoreSize().getOffHeapSize() , this.flushSizeLowerBound ); return true; } return false; } }
default long decMemStoreSize(MemStoreSize delta) { return incMemStoreSize(-delta.getDataSize(), -delta.getHeapSize(), -delta.getOffHeapSize(), -delta.getCellsCount()); }
void decrMemStoreSize(MemStoreSize mss) { decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
default long incMemStoreSize(MemStoreSize delta) { return incMemStoreSize(delta.getDataSize(), delta.getHeapSize(), delta.getOffHeapSize(), delta.getCellsCount()); }
protected Segment(CellComparator comparator, List<ImmutableSegment> segments, TimeRangeTracker trt) { long dataSize = 0; long heapSize = 0; long OffHeapSize = 0; int cellsCount = 0; for (Segment segment : segments) { MemStoreSize memStoreSize = segment.getMemStoreSize(); dataSize += memStoreSize.getDataSize(); heapSize += memStoreSize.getHeapSize(); OffHeapSize += memStoreSize.getOffHeapSize(); cellsCount += memStoreSize.getCellsCount(); } this.comparator = comparator; this.updatesLock = new ReentrantReadWriteLock(); // Do we need to be thread safe always? What if ImmutableSegment? // DITTO for the TimeRangeTracker below. this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount); this.timeRangeTracker = trt; }
mt.storeFileCount += store.getStorefilesCount(); mt.memstoreSize += (store.getMemStoreSize().getDataSize() + store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize()); mt.storeFileSize += store.getStorefilesSize(); mt.referenceFileCount += store.getNumReferenceFiles();
region.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount());
private long addRowsByKeysDataSize(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); MemStoreSizing memstoreSizing = new NonThreadSafeMemStoreSizing(); for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); hmc.add(kv, memstoreSizing); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); } MemStoreSize mss = memstoreSizing.getMemStoreSize(); regionServicesForStores.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); return mss.getDataSize(); }
void checkResources() throws RegionTooBusyException { // If catalog region, do not impose resource constraints or block updates. if (this.getRegionInfo().isMetaRegion()) return; MemStoreSize mss = this.memStoreSizing.getMemStoreSize(); if (mss.getHeapSize() + mss.getOffHeapSize() > this.blockingMemStoreSize) { blockedRequestsCount.increment(); requestFlush(); // Don't print current limit because it will vary too much. The message is used as a key // over in RetriesExhaustedWithDetailsException processing. throw new RegionTooBusyException("Over memstore limit=" + org.apache.hadoop.hbase.procedure2.util.StringUtils.humanSize(this.blockingMemStoreSize) + ", regionName=" + (this.getRegionInfo() == null? "unknown": this.getRegionInfo().getEncodedName()) + ", server=" + (this.getRegionServerServices() == null? "unknown": this.getRegionServerServices().getServerName())); } }
region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); ImmutableSegment s = memstore.getSnapshot();
region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); ImmutableSegment s = memstore.getSnapshot();
region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); ImmutableSegment s = memstore.getSnapshot();