@Override public MemStoreSize getMemStoreSize() { return new MemStoreSize(this.dataSize, this.heapSize, this.offHeapSize, this.cellsCount); }
/** * Package private constructor. */ MemStoreSize(MemStoreSize memStoreSize) { this.dataSize = memStoreSize.getDataSize(); this.heapSize = memStoreSize.getHeapSize(); this.offHeapSize = memStoreSize.getOffHeapSize(); this.cellsCount = memStoreSize.getCellsCount(); }
@Override public String toString() { return getMemStoreSize().toString(); } }
void incGlobalMemStoreSize(MemStoreSize mss) { incGlobalMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); }
private boolean isFlushSize(MemStoreSize size) { return size.getHeapSize() + size.getOffHeapSize() > getMemStoreFlushSize(); }
LOG.info("memstore1 estimated size={}", memStoreSizing.getMemStoreSize().getDataSize() + memStoreSizing.getMemStoreSize().getHeapSize()); for (int i = 0; i < count; i++) { memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memStoreSizing); memStoreSizing.getMemStoreSize().getDataSize() + memStoreSizing.getMemStoreSize().getHeapSize()); memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]), memStoreSizing); LOG.info("memstore2 estimated size={}", memStoreSizing.getMemStoreSize().getDataSize() + memStoreSizing.getMemStoreSize().getHeapSize()); final int seconds = 30; LOG.info("Waiting " + seconds + " seconds while heap dump is taken");
@Override public long getDataSize() { return this.mss.getDataSize(); }
@Override public long getHeapSize() { return this.mss.getHeapSize(); }
@Override public int getCellsCount() { return this.mss.getCellsCount(); }
@Override public long getOffHeapSize() { return this.mss.getOffHeapSize(); }
long memstoresize = this.memStoreSizing.getMemStoreSize().getDataSize(); String msg = "Finished flush of" + " dataSize ~" + StringUtils.byteDesc(mss.getDataSize()) + "/" + mss.getDataSize() + ", heapSize ~" + StringUtils.byteDesc(mss.getHeapSize()) + "/" + mss.getHeapSize() + ", currentSize=" + StringUtils.byteDesc(memstoresize) + "/" + memstoresize + " for " + this.getRegionInfo().getEncodedName() + " in " + time + "ms, sequenceid=" rsServices.getMetrics().updateFlush(getTableDescriptor().getTableName().getNameAsString(), time, mss.getDataSize(), flushedOutputFileSize);
/** * Utility method broken out of internalPrepareFlushCache so that method is smaller. */ private void logFatLineOnFlush(Collection<HStore> storesToFlush, long sequenceId) { if (!LOG.isInfoEnabled()) { return; } // Log a fat line detailing what is being flushed. StringBuilder perCfExtras = null; if (!isAllFamilies(storesToFlush)) { perCfExtras = new StringBuilder(); for (HStore store: storesToFlush) { MemStoreSize mss = store.getFlushableSize(); perCfExtras.append("; ").append(store.getColumnFamilyName()); perCfExtras.append("={dataSize=") .append(StringUtils.byteDesc(mss.getDataSize())); perCfExtras.append(", heapSize=") .append(StringUtils.byteDesc(mss.getHeapSize())); perCfExtras.append(", offHeapSize=") .append(StringUtils.byteDesc(mss.getOffHeapSize())); perCfExtras.append("}"); } } MemStoreSize mss = this.memStoreSizing.getMemStoreSize(); LOG.info("Flushing " + + storesToFlush.size() + "/" + stores.size() + " column families," + " dataSize=" + StringUtils.byteDesc(mss.getDataSize()) + " heapSize=" + StringUtils.byteDesc(mss.getHeapSize()) + ((perCfExtras != null && perCfExtras.length() > 0)? perCfExtras.toString(): "") + ((wal != null) ? "" : "; WAL is null, using passed sequenceid=" + sequenceId)); }
public long getDataSize() { return memStoreSize.getDataSize(); }
@Override public long heapSize() { MemStoreSize memstoreSize = this.memstore.size(); return DEEP_OVERHEAD + memstoreSize.getHeapSize(); }
protected boolean shouldFlush(HStore store) { if (store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize() > this.flushSizeLowerBound) { LOG.debug("Flush {} of {}; " + "heap memstoreSize={} +" + "off heap memstoreSize={} > memstore lowerBound={}" , store.getColumnFamilyName() , region.getRegionInfo().getEncodedName() , store.getMemStoreSize().getHeapSize() , store.getMemStoreSize().getOffHeapSize() , this.flushSizeLowerBound ); return true; } return false; } }
ThreadSafeMemStoreSizing(MemStoreSize mss) { this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); }
assertTrue(cf1MemstoreSizePhaseI.getDataSize() > 0); assertTrue(cf2MemstoreSizePhaseI.getDataSize() > 0); assertTrue(cf3MemstoreSizePhaseI.getDataSize() > 0); cf1MemstoreSizePhaseI.getDataSize() + cf2MemstoreSizePhaseI.getDataSize() + cf3MemstoreSizePhaseI.getDataSize()); assertTrue(cf1MemstoreSizePhaseI.getHeapSize() > cf1MemstoreSizePhaseII.getHeapSize()); assertEquals(cf1MemstoreSizePhaseI.getDataSize(), cf1MemstoreSizePhaseII.getDataSize()); assertEquals(0, cf2MemstoreSizePhaseII.getDataSize()); assertEquals(2*cf1MemstoreSizePhaseI.getDataSize(), cf1MemstoreSizePhaseIV.getDataSize()); cf1MemstoreSizePhaseI.getHeapSize() - cf1MemstoreSizePhaseII.getHeapSize(), cf1MemstoreSizePhaseIII.getHeapSize() - cf1MemstoreSizePhaseIV.getHeapSize() - CellArrayImmutableSegment.DEEP_OVERHEAD_CAM); assertEquals(3, // active, one in pipeline, snapshot + " CF1: " + cf1MemstoreSizePhaseI.getDataSize() + "/" + cf1MemstoreSizePhaseII .getDataSize() + "--" + cf1MemstoreSizePhaseI.getHeapSize() + "/" + cf1MemstoreSizePhaseII .getHeapSize() + ", CF2: " + cf2MemstoreSizePhaseI.getDataSize() + "/" + cf2MemstoreSizePhaseII.getDataSize() + "--" + cf2MemstoreSizePhaseI.getHeapSize() + "/" + cf2MemstoreSizePhaseII.getHeapSize() + ", CF3: " + cf3MemstoreSizePhaseI.getDataSize() + "/" + cf3MemstoreSizePhaseII.getDataSize() + "--" + cf3MemstoreSizePhaseI.getHeapSize() + "/" + cf3MemstoreSizePhaseII.getHeapSize() + "\n<<< AND before/after second flushes " + " CF1: " + cf1MemstoreSizePhaseIII.getDataSize() + "/" + cf1MemstoreSizePhaseIV .getDataSize() + "--" + cf1MemstoreSizePhaseIII.getHeapSize() + "/" + cf1MemstoreSizePhaseIV
for (Store store : r.getStores()) { mt.storeFileCount += store.getStorefilesCount(); mt.memstoreSize += (store.getMemStoreSize().getDataSize() + store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize()); mt.storeFileSize += store.getStorefilesSize(); mt.referenceFileCount += store.getNumReferenceFiles();
void throwException(String title, String regionName) { StringBuilder buf = new StringBuilder(); buf.append(title + ", "); buf.append(getRegionInfo().toString()); buf.append(getRegionInfo().isMetaRegion() ? " meta region " : " "); buf.append("stores: "); for (HStore s : stores.values()) { buf.append(s.getColumnFamilyDescriptor().getNameAsString()); buf.append(" size: "); buf.append(s.getMemStoreSize().getDataSize()); buf.append(" "); } buf.append("end-of-stores"); buf.append(", memstore size "); buf.append(getMemStoreDataSize()); if (getRegionInfo().getRegionNameAsString().startsWith(regionName)) { throw new RuntimeException(buf.toString()); } }
/** * @return statistics about the current load of the region */ public ClientProtos.RegionLoadStats getLoadStatistics() { if (!regionStatsEnabled) { return null; } ClientProtos.RegionLoadStats.Builder stats = ClientProtos.RegionLoadStats.newBuilder(); stats.setMemStoreLoad((int) (Math.min(100, (this.memStoreSizing.getMemStoreSize().getHeapSize() * 100) / this.memstoreFlushSize))); if (rsServices.getHeapMemoryManager() != null) { // the HeapMemoryManager uses -0.0 to signal a problem asking the JVM, // so we could just do the calculation below and we'll get a 0. // treating it as a special case analogous to no HMM instead so that it can be // programatically treated different from using <1% of heap. final float occupancy = rsServices.getHeapMemoryManager().getHeapOccupancyPercent(); if (occupancy != HeapMemoryManager.HEAP_OCCUPANCY_ERROR_VALUE) { stats.setHeapOccupancy((int)(occupancy * 100)); } } stats.setCompactionPressure((int) (rsServices.getCompactionPressure() * 100 > 100 ? 100 : rsServices.getCompactionPressure() * 100)); return stats.build(); }