@Test public void testOffheapMemstoreHigherWaterMarkLimitsDueToHeapSize() { // setting 1G as offheap data size conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1L * 1024L)); // try for default cases RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); // this will breach higher limit as heap size is higher and not due to offheap size long dataSize = regionServerAccounting.getGlobalOnHeapMemStoreLimit(); MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 100); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK, regionServerAccounting.isAboveHighWaterMark()); }
if (!blocked) { startTime = EnvironmentEdgeManager.currentTime(); if (!server.getRegionServerAccounting().isOffheap()) { logMsg("global memstore heapsize", server.getRegionServerAccounting().getGlobalMemStoreHeapSize(), server.getRegionServerAccounting().getGlobalMemStoreLimit()); } else { switch (flushType) { case ABOVE_OFFHEAP_HIGHER_MARK: logMsg("the global offheap memstore datasize", server.getRegionServerAccounting().getGlobalMemStoreOffHeapSize(), server.getRegionServerAccounting().getGlobalMemStoreLimit()); break; case ABOVE_ONHEAP_HIGHER_MARK: logMsg("global memstore heapsize", server.getRegionServerAccounting().getGlobalMemStoreHeapSize(), server.getRegionServerAccounting().getGlobalOnHeapMemStoreLimit()); break; default:
/** * @return the flush pressure of all stores on this regionserver. The value should be greater than * or equal to 0.0, and any value greater than 1.0 means we enter the emergency state that * global memstore size already exceeds lower limit. */ public double getFlushPressure() { if (memType == MemoryType.HEAP) { return (getGlobalMemStoreHeapSize()) * 1.0 / globalMemStoreLimitLowMark; } else { return Math.max(getGlobalMemStoreOffHeapSize() * 1.0 / globalMemStoreLimitLowMark, getGlobalMemStoreHeapSize() * 1.0 / globalOnHeapMemstoreLimitLowMark); } } }
/** * @param conf * @param server */ public MemStoreFlusher(final Configuration conf, final HRegionServer server) { super(); this.conf = conf; this.server = server; this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); this.blockingWaitTime = conf.getInt("hbase.hstore.blockingWaitTime", 90000); int handlerCount = conf.getInt("hbase.hstore.flusher.count", 2); this.flushHandlers = new FlushHandler[handlerCount]; LOG.info("globalMemStoreLimit=" + TraditionalBinaryPrefix .long2String(this.server.getRegionServerAccounting().getGlobalMemStoreLimit(), "", 1) + ", globalMemStoreLimitLowMark=" + TraditionalBinaryPrefix.long2String( this.server.getRegionServerAccounting().getGlobalMemStoreLimitLowMark(), "", 1) + ", Offheap=" + (this.server.getRegionServerAccounting().isOffheap())); }
@Test public void testOnheapMemstoreLowerWaterMarkLimits() { RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); long dataSize = regionServerAccounting.getGlobalMemStoreLimit(); MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 0); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark()); }
@Test public void testOnheapMemstoreHigherWaterMarkLimits() { RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); long dataSize = regionServerAccounting.getGlobalMemStoreLimit(); MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 0); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK, regionServerAccounting.isAboveHighWaterMark()); }
@Test public void testOffheapMemstoreLowerWaterMarkLimitsDueToHeapSize() { // setting 1G as offheap data size conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1L * 1024L)); // try for default cases RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); // this will breach higher limit as heap size is higher and not due to offheap size long dataSize = regionServerAccounting.getGlobalOnHeapMemStoreLimit(); MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 100); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark()); } }
@Test public void testOffheapMemstoreLowerWaterMarkLimitsDueToDataSize() { // setting 1G as offheap data size conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1L * 1024L)); // try for default cases RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); // this will breach offheap limit as data size is higher and not due to heap size MemStoreSize memstoreSize = new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L), 100); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); assertEquals(FlushType.ABOVE_OFFHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark()); }
@Test public void testOffheapMemstoreHigherWaterMarkLimitsDueToDataSize() { // setting 1G as offheap data size conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1L * 1024L)); // try for default cases RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); // this will breach offheap limit as data size is higher and not due to heap size MemStoreSize memstoreSize = new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L), 100); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); assertEquals(FlushType.ABOVE_OFFHEAP_HIGHER_MARK, regionServerAccounting.isAboveHighWaterMark()); }
regionServerAccounting = new RegionServerAccounting(conf);
@Override public boolean progress() { replayedEdits++; //during replay, rsAccounting should align with global memstore, because //there is only one memstore here Assert.assertEquals(rsAccounting.getGlobalMemStoreDataSize(), region.getMemStoreDataSize()); Assert.assertEquals(rsAccounting.getGlobalMemStoreHeapSize(), region.getMemStoreHeapSize()); Assert.assertEquals(rsAccounting.getGlobalMemStoreOffHeapSize(), region.getMemStoreOffHeapSize()); // abort the replay before finishing, leaving some edits in the memory return replayedEdits < totalEdits - 10; } });
CONF.setInt("hbase.hstore.report.period", 0); final RegionServerAccounting rsAccounting = new RegionServerAccounting(CONF); RegionServerServices rs = Mockito.mock(RegionServerServices.class); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); Assert.assertEquals(0, rsAccounting.getGlobalMemStoreDataSize()); Assert.assertEquals(0, region.getMemStoreDataSize());
/** * Return true if we're above the high watermark */ private boolean isAboveLowWaterMark() { return server.getRegionServerAccounting(). getGlobalMemstoreSize() >= globalMemStoreLimitLowMark; }
/** * Roll back the global MemStore size for a specified region when this region * can't be opened. * * @param regionName the region which could not open. * @return the global Memstore size in the RegionServer */ public long rollbackRegionReplayEditsSize(byte[] regionName) { AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName); long editsSizeLong = 0L; if (replayEditsSize != null) { editsSizeLong = -replayEditsSize.get(); clearRegionReplayEditsSize(regionName); } return addAndGetGlobalMemstoreSize(editsSizeLong); }
void incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, int cellsCountDelta) { if (this.rsAccounting != null) { rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta); } long dataSize = this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta, cellsCountDelta); checkNegativeMemStoreDataSize(dataSize, dataSizeDelta); }
/** * Increase the size of mem store in this region and the size of global mem * store * @param memStoreSize * @return the size of memstore in this region */ public long addAndGetGlobalMemstoreSize(long memStoreSize) { if (this.rsAccounting != null) { rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize); } return this.memstoreSize.getAndAdd(memStoreSize); }
/** * Used by tests * @param s Store to add edit too. * @param kv KeyValue to add. * @return True if we should flush. */ protected boolean restoreEdit(final Store s, final KeyValue kv) { long kvSize = s.add(kv); if (this.rsAccounting != null) { rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize); } return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize)); }
this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
@Override public long getMemStoreLimit() { return this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit(); }
long globalMemstoreHeapSize = regionServerAccounting.getGlobalMemStoreHeapSize(); tunerContext.setCurMemStoreUsed((float) globalMemstoreHeapSize / maxHeapSize); metricsHeapMemoryManager.setCurMemStoreSizeGauge(globalMemstoreHeapSize);