@Override public StorageProperties newStorageProperties() { return new StorageProperties(propertiesAccessor); }
public int getBufferCacheNumPages() { return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES)); }
public int getMemoryComponentNumPages() { final long metadataReservedMem = getMetadataReservedMemory(); final long globalUserDatasetMem = getMemoryComponentGlobalBudget() - metadataReservedMem; final long userDatasetMem = globalUserDatasetMem / (getMaxActiveWritableDatasets() + geSystemReservedDatasets()); return (int) (userDatasetMem / getMemoryComponentPageSize()); }
public long getJobExecutionMemoryBudget() { final long jobExecutionMemory = Runtime.getRuntime().maxMemory() - getBufferCacheSize() - getMemoryComponentGlobalBudget(); if (jobExecutionMemory <= 0) { final String msg = String.format( "Invalid node memory configuration, more memory budgeted than available in JVM. Runtime max memory:" + " (%d), Buffer cache memory (%d), memory component global budget (%d)", Runtime.getRuntime().maxMemory(), getBufferCacheSize(), getMemoryComponentGlobalBudget()); throw new IllegalStateException(msg); } return jobExecutionMemory; }
ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider(); double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(); ITypeTraits[] typeTraits = getInvListTypeTraits(mdProvider, dataset, recordType, metaType); IBinaryComparatorFactory[] cmpFactories =
IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000); IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator, storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages()); lsmIOScheduler = new AsynchronousScheduler(getServiceContext().getThreadFactory(), HaltCallback.INSTANCE); metadataMergePolicyFactory = new PrefixMergePolicyFactory(); storageProperties.getBufferCacheMaxOpenFiles(), getServiceContext().getThreadFactory(), replicationManager); } else { bufferCache = new BufferCache(ioManager, prs, pcp, new FileMapManager(), storageProperties.getBufferCacheMaxOpenFiles(), getServiceContext().getThreadFactory());
public CompressionManager(StorageProperties storageProperties) { validateCompressionConfiguration(storageProperties); defaultScheme = storageProperties.getCompressionScheme(); }
@Override public NodeCapacity getCapacity() { StorageProperties storageProperties = runtimeContext.getStorageProperties(); final long memorySize = storageProperties.getJobExecutionMemoryBudget(); int allCores = Runtime.getRuntime().availableProcessors(); return new NodeCapacity(memorySize, allCores); }
btreeCompFactories, valueProviderFactories, rTreePolicyType, linearizeCmpFactory, rtreeFields, new int[] { numNestedSecondaryKeyFields }, isPointMBR, mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate());
/** * Validate the configuration of StorageProperties * * @param storageProperties */ private void validateCompressionConfiguration(StorageProperties storageProperties) { if (!isRegisteredScheme(storageProperties.getCompressionScheme())) { final String option = StorageProperties.Option.STORAGE_COMPRESSION_BLOCK.ini(); final String value = storageProperties.getCompressionScheme(); throw new IllegalStateException("Invalid compression configuration (" + option + " = " + value + "). Valid values are: " + formatSupportedValues()); } }
IBinaryComparatorFactory[] cmpFactories = getCmpFactories(mdProvider, dataset, index, recordType, metaType); int[] bloomFilterFields = getBloomFilterFields(dataset, index); double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(); ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index); ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index);
externalProperties = new ExternalProperties(propertiesAccessor); metadataProperties = new MetadataProperties(propertiesAccessor); storageProperties = new StorageProperties(propertiesAccessor); txnProperties = new TransactionProperties(propertiesAccessor); activeProperties = new ActiveProperties(propertiesAccessor);
if (isNewUniverse()) { final double bloomFilterFalsePositiveRate = appContext.getStorageProperties().getBloomFilterFalsePositiveRate(); LSMBTreeLocalResourceFactory lsmBtreeFactory = new LSMBTreeLocalResourceFactory( storageComponentProvider.getStorageManager(), typeTraits, cmpFactories, null, null, null,
@Override public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException { int[] filterFields = IndexUtil.getFilterFields(dataset, index, filterTypeTraits); int[] btreeFields = IndexUtil.getBtreeFieldsIfFiltered(dataset, index); IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider(); ITypeTraits[] typeTraits = getTypeTraits(mdProvider, dataset, index, recordType, metaType); IBinaryComparatorFactory[] cmpFactories = getCmpFactories(mdProvider, dataset, index, recordType, metaType); int[] bloomFilterFields = getBloomFilterFields(dataset, index); double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(); ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index); if (opTrackerFactory instanceof PrimaryIndexOperationTrackerFactory) { opTrackerFactory = new TestPrimaryIndexOperationTrackerFactory(dataset.getDatasetId()); } ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index); IStorageManager storageManager = storageComponentProvider.getStorageManager(); IMetadataPageManagerFactory metadataPageManagerFactory = storageComponentProvider.getMetadataPageManagerFactory(); ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider(); AsterixVirtualBufferCacheProvider vbcProvider = new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()); return new TestLsmBtreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, true, bloomFilterFields, bloomFilterFalsePositiveRate, index.isPrimaryIndex(), btreeFields); }