/** {@inheritDoc} */ @Override public int getMaxSize() { return (int) (dataRegCfg.getMaxSize() / (1024 * 1024)); }
/** * Get checkpoint buffer size for the given configuration. * * @param regCfg Configuration. * @return Checkpoint buffer size. */ public static long checkpointBufferSize(DataRegionConfiguration regCfg) { if (!regCfg.isPersistenceEnabled()) return 0L; long res = regCfg.getCheckpointPageBufferSize(); if (res == 0L) { if (regCfg.getMaxSize() < GB) res = Math.min(DFLT_MIN_CHECKPOINTING_PAGE_BUFFER_SIZE, regCfg.getMaxSize()); else if (regCfg.getMaxSize() < 8 * GB) res = regCfg.getMaxSize() / 4; else res = DFLT_MAX_CHECKPOINTING_PAGE_BUFFER_SIZE; } return res; }
/** * @return Configured data regions offheap memory in bytes. */ private long configuredOffheap() { DataStorageConfiguration memCfg = ctx.config().getDataStorageConfiguration(); if (memCfg == null) return 0; long res = memCfg.getDefaultDataRegionConfiguration().getMaxSize(); DataRegionConfiguration[] dataRegions = memCfg.getDataRegionConfigurations(); if (dataRegions != null) { for (DataRegionConfiguration dataReg : dataRegions) res += dataReg.getMaxSize(); } return res; }
/** {@inheritDoc} */ @Override public void start() throws IgniteException { long startSize = dataRegionCfg.getInitialSize(); long maxSize = dataRegionCfg.getMaxSize(); long[] chunks = new long[SEG_CNT]; chunks[0] = startSize; long total = startSize; long allocChunkSize = Math.max((maxSize - startSize) / (SEG_CNT - 1), 256L * 1024 * 1024); int lastIdx = 0; for (int i = 1; i < SEG_CNT; i++) { long allocSize = Math.min(allocChunkSize, maxSize - total); if (allocSize <= 0) break; chunks[i] = allocSize; total += allocSize; lastIdx = i; } if (lastIdx != SEG_CNT - 1) chunks = Arrays.copyOf(chunks, lastIdx + 1); if (segments == null) directMemoryProvider.initialize(chunks); addSegment(null); }
/** * @return Maximum number of bytes for IGFS data cache. */ public long maxSpaceSize() { DataRegion plc = dataCachePrj.context().dataRegion(); long size = plc != null ? plc.config().getMaxSize() : 0; return (size <= 0) ? 0 : size ; }
/** * @param regionCfg Region config. */ private void validateDataRegion(DataRegionConfiguration regionCfg) { if (regionCfg.isPersistenceEnabled() && regionCfg.getMaxSize() == DataStorageConfiguration.DFLT_DATA_REGION_MAX_SIZE) throw new AssertionError("Max size of data region should be set explicitly to avoid memory over usage"); }
/** * @param pageMem Page memory. * @param plcCfg Policy config. * @param sharedCtx Shared context. */ public RandomLruPageEvictionTracker( PageMemory pageMem, DataRegionConfiguration plcCfg, GridCacheSharedContext<?, ?> sharedCtx ) { super((PageMemoryNoStoreImpl)pageMem, plcCfg, sharedCtx); DataStorageConfiguration memCfg = sharedCtx.kernalContext().config().getDataStorageConfiguration(); assert plcCfg.getMaxSize() / memCfg.getPageSize() < Integer.MAX_VALUE; log = sharedCtx.logger(getClass()); }
/** * @param pageMem Page memory. * @param plcCfg Policy config. * @param sharedCtx Shared context. */ public Random2LruPageEvictionTracker( PageMemoryNoStoreImpl pageMem, DataRegionConfiguration plcCfg, GridCacheSharedContext<?, ?> sharedCtx ) { super(pageMem, plcCfg, sharedCtx); DataStorageConfiguration memCfg = sharedCtx.kernalContext().config().getDataStorageConfiguration(); assert plcCfg.getMaxSize() / memCfg.getPageSize() < Integer.MAX_VALUE; log = sharedCtx.logger(getClass()); }
/** * @param regCfg DataRegionConfiguration to validate. * @throws IgniteCheckedException If config is invalid. */ private void checkDataRegionSize(DataRegionConfiguration regCfg) throws IgniteCheckedException { if (regCfg.getInitialSize() < MIN_PAGE_MEMORY_SIZE || regCfg.getMaxSize() < MIN_PAGE_MEMORY_SIZE) throw new IgniteCheckedException("DataRegion must have size more than 10MB (use " + "DataRegionConfiguration.initialSize and .maxSize properties to set correct size in bytes) " + "[name=" + regCfg.getName() + ", initialSize=" + U.readableSize(regCfg.getInitialSize(), true) + ", maxSize=" + U.readableSize(regCfg.getMaxSize(), true) + "]" ); if (regCfg.getMaxSize() < regCfg.getInitialSize()) { if (regCfg.getInitialSize() != Math.min(DataStorageConfiguration.DFLT_DATA_REGION_MAX_SIZE, DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)) { throw new IgniteCheckedException("DataRegion maxSize must not be smaller than initialSize" + "[name=" + regCfg.getName() + ", initialSize=" + U.readableSize(regCfg.getInitialSize(), true) + ", maxSize=" + U.readableSize(regCfg.getMaxSize(), true) + "]"); } regCfg.setInitialSize(regCfg.getMaxSize()); LT.warn(log, "DataRegion maxSize=" + U.readableSize(regCfg.getMaxSize(), true) + " is smaller than defaultInitialSize=" + U.readableSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE, true) + ", setting initialSize to " + U.readableSize(regCfg.getMaxSize(), true)); } if (U.jvm32Bit() && regCfg.getInitialSize() > MAX_PAGE_MEMORY_INIT_SIZE_32_BIT) throw new IgniteCheckedException("DataRegion initialSize exceeds 2GB on 32-bit JVM (use " + "DataRegionConfiguration.initialSize property to set correct size in bytes or use 64-bit JVM) " + "[name=" + regCfg.getName() + ", size=" + U.readableSize(regCfg.getInitialSize(), true) + "]"); }
/** * @return Required offheap memory in bytes. */ private long requiredOffheap() { if(ctx.config().isClientMode()) return 0; DataStorageConfiguration memCfg = ctx.config().getDataStorageConfiguration(); assert memCfg != null; long res = memCfg.getSystemRegionMaxSize(); // Add memory policies. DataRegionConfiguration[] dataRegions = memCfg.getDataRegionConfigurations(); if (dataRegions != null) { for (DataRegionConfiguration dataReg : dataRegions) { res += dataReg.getMaxSize(); res += U.checkpointBufferSize(dataReg); } } res += memCfg.getDefaultDataRegionConfiguration().getMaxSize(); res += U.checkpointBufferSize(memCfg.getDefaultDataRegionConfiguration()); return res; }
/** * Constructor. * * @param plc Memory policy configuration. */ public VisorMemoryPolicyConfiguration(DataRegionConfiguration plc) { assert plc != null; name = plc.getName(); maxSize = plc.getMaxSize(); initSize = plc.getInitialSize(); swapFilePath = plc.getSwapPath(); pageEvictionMode = plc.getPageEvictionMode(); evictionThreshold = plc.getEvictionThreshold(); emptyPagesPoolSize = plc.getEmptyPagesPoolSize(); }
/** * Constructor. * * @param plc Data region configuration. */ public VisorDataRegionConfiguration(DataRegionConfiguration plc) { assert plc != null; name = plc.getName(); initSize = plc.getInitialSize(); maxSize = plc.getMaxSize(); swapPath = plc.getSwapPath(); pageEvictionMode = plc.getPageEvictionMode(); evictionThreshold = plc.getEvictionThreshold(); emptyPagesPoolSize = plc.getEmptyPagesPoolSize(); metricsEnabled = plc.isMetricsEnabled(); metricsSubIntervalCount = plc.getMetricsSubIntervalCount(); metricsRateTimeInterval = plc.getMetricsRateTimeInterval(); persistenceEnabled = plc.isPersistenceEnabled(); checkpointPageBufSize = plc.getCheckpointPageBufferSize(); }
/** * See {@link GridCacheMapEntry#ensureFreeSpace()} * * @param memPlc data region. */ public void ensureFreeSpace(DataRegion memPlc) throws IgniteCheckedException { if (memPlc == null) return; DataRegionConfiguration plcCfg = memPlc.config(); if (plcCfg.getPageEvictionMode() == DataPageEvictionMode.DISABLED || plcCfg.isPersistenceEnabled()) return; long memorySize = plcCfg.getMaxSize(); PageMemory pageMem = memPlc.pageMemory(); int sysPageSize = pageMem.systemPageSize(); CacheFreeListImpl freeListImpl = freeListMap.get(plcCfg.getName()); for (;;) { long allocatedPagesCnt = pageMem.loadedPages(); int emptyDataPagesCnt = freeListImpl.emptyDataPages(); boolean shouldEvict = allocatedPagesCnt > (memorySize / sysPageSize * plcCfg.getEvictionThreshold()) && emptyDataPagesCnt < plcCfg.getEmptyPagesPoolSize(); if (shouldEvict) { warnFirstEvict(plcCfg); memPlc.evictionTracker().evictDataPage(); memPlc.memoryMetrics().updateEvictionRate(); } else break; } }
/** * @param regCfg DataRegionConfiguration to validate. * @param dbCfg Memory configuration. * @throws IgniteCheckedException If config is invalid. */ protected void checkRegionEvictionProperties(DataRegionConfiguration regCfg, DataStorageConfiguration dbCfg) throws IgniteCheckedException { if (regCfg.getPageEvictionMode() == DataPageEvictionMode.DISABLED) return; if (regCfg.getEvictionThreshold() < 0.5 || regCfg.getEvictionThreshold() > 0.999) { throw new IgniteCheckedException("Page eviction threshold must be between 0.5 and 0.999: " + regCfg.getName()); } if (regCfg.getEmptyPagesPoolSize() <= 10) throw new IgniteCheckedException("Evicted pages pool size should be greater than 10: " + regCfg.getName()); long maxPoolSize = regCfg.getMaxSize() / dbCfg.getPageSize() / 10; if (regCfg.getEmptyPagesPoolSize() >= maxPoolSize) { throw new IgniteCheckedException("Evicted pages pool size should be lesser than " + maxPoolSize + ": " + regCfg.getName()); } }
/** * @param regCfg Data region configuration. * @return Data region message. */ private String dataRegionConfigurationMessage(DataRegionConfiguration regCfg) { if (regCfg == null) return null; SB m = new SB(); m.a(" ^-- ").a(regCfg.getName()).a(" ["); m.a("initSize=").a(U.readableSize(regCfg.getInitialSize(), false)); m.a(", maxSize=").a(U.readableSize(regCfg.getMaxSize(), false)); m.a(", persistence=" + regCfg.isPersistenceEnabled()).a(']'); return m.toString(); }
/** * User is allowed to configure memory policy with 'default' name, * in that case Ignite instance will use this user-defined memory policy as a default one. */ @Test public void testCustomConfigOverridesDefault() throws Exception { prepareCustomConfigWithOverridingDefault(); IgniteEx ignite = startGrid(0); IgniteCacheDatabaseSharedManager dbMgr = ignite.context().cache().context().database(); Collection<DataRegion> allMemPlcs = dbMgr.dataRegions(); assertEquals(3, allMemPlcs.size()); verifyDefaultAndSystemMemoryPolicies(allMemPlcs); DataRegion dfltMemPlc = U.field(dbMgr, "dfltDataRegion"); assertEquals(dfltMemPlc.config().getMaxSize(), USER_DEFAULT_MEM_PLC_SIZE); }
/** * User is allowed to define fully custom memory policy and make it default by setting its name to memory config. * * At the same time user still can create a memory policy with name 'default' * which although won't be used as default. */ @Test public void testCustomConfigOverridesDefaultNameAndDeclaresDefault() throws Exception { prepareCustomConfigWithOverriddenDefaultName(); IgniteEx ignite = startGrid(0); IgniteCacheDatabaseSharedManager dbMgr = ignite.context().cache().context().database(); Collection<DataRegion> allMemPlcs = dbMgr.dataRegions(); assertEquals(4, allMemPlcs.size()); verifyDefaultAndSystemMemoryPolicies(allMemPlcs); DataRegion dfltMemPlc = U.field(dbMgr, "dfltDataRegion"); assertEquals(dfltMemPlc.config().getMaxSize(), USER_CUSTOM_MEM_PLC_SIZE); }
/** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); boolean client = igniteInstanceName.startsWith("client"); DataStorageConfiguration dsCfg = new DataStorageConfiguration(); if (!client) { DataRegionConfiguration drCfg1 = new DataRegionConfiguration(); drCfg1.setMaxSize(16 * 1024 * 1024); drCfg1.setName("nopersistence"); drCfg1.setInitialSize(drCfg1.getMaxSize()); drCfg1.setPersistenceEnabled(false); DataRegionConfiguration drCfg2 = new DataRegionConfiguration(); drCfg2.setMaxSize(16 * 1024 * 1024); drCfg2.setName("persistence"); drCfg2.setInitialSize(drCfg2.getMaxSize()); drCfg2.setPersistenceEnabled(true); dsCfg.setDataRegionConfigurations(drCfg1, drCfg2); cfg.setDataStorageConfiguration(dsCfg); } else { CacheConfiguration ccfg1 = new CacheConfiguration(PERSISTED_CACHE); CacheConfiguration ccfg2 = new CacheConfiguration(INMEMORY_CACHE); ccfg1.setDataRegionName("persistence"); ccfg2.setDataRegionName("nopersistence"); cfg.setCacheConfiguration(ccfg1, ccfg2); cfg.setClientMode(true); } return cfg; }
/** * Writes the data region configuration. * * @param w Writer. */ private static void writeDataRegionConfiguration(BinaryRawWriter w, DataRegionConfiguration cfg) { assert w != null; assert cfg != null; w.writeString(cfg.getName()); w.writeBoolean(cfg.isPersistenceEnabled()); w.writeLong(cfg.getInitialSize()); w.writeLong(cfg.getMaxSize()); w.writeString(cfg.getSwapPath()); w.writeInt(cfg.getPageEvictionMode().ordinal()); w.writeDouble(cfg.getEvictionThreshold()); w.writeInt(cfg.getEmptyPagesPoolSize()); w.writeBoolean(cfg.isMetricsEnabled()); w.writeInt(cfg.getMetricsSubIntervalCount()); w.writeLong(cfg.getMetricsRateTimeInterval()); w.writeLong(cfg.getCheckpointPageBufferSize()); }
/** * Verifies that {@link IgniteCheckedException} is thrown when empty pages pool size is greater than * DataRegionConfiguration.getMaxSize() / DataStorageConfiguration.getPageSize() / 10. */ @Test public void testInvalidBigEmptyPagesPoolSize() { final int DFLT_PAGE_SIZE = 1024; long expectedMaxPoolSize; DataRegionConfiguration invCfg = new DataRegionConfiguration(); invCfg.setName("invCfg"); invCfg.setInitialSize(DFLT_MEM_PLC_SIZE); invCfg.setMaxSize(DFLT_MEM_PLC_SIZE); invCfg.setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU); memCfg = new DataStorageConfiguration(); memCfg.setDataRegionConfigurations(invCfg); memCfg.setPageSize(DFLT_PAGE_SIZE); ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); expectedMaxPoolSize = invCfg.getMaxSize() / memCfg.getPageSize() / 10; if (expectedMaxPoolSize < Integer.MAX_VALUE) { // Setting the empty pages pool size greater than // DataRegionConfiguration.getMaxSize() / DataStorageConfiguration.getPageSize() / 10 invCfg.setEmptyPagesPoolSize((int)expectedMaxPoolSize + 1); memCfg.setDataRegionConfigurations(invCfg); checkStartGridException(IgniteCheckedException.class, "Failed to start processor: GridProcessorAdapter []"); } }