public RocksDbCacheOperator(TopologyContext context, String cacheDir) { this.stormConf = context.getStormConf(); this.maxFlushSize = ConfigExtension.getTransactionCacheBatchFlushSize(stormConf); Options rocksDbOpt = new Options(); rocksDbOpt.setCreateMissingColumnFamilies(true).setCreateIfMissing(true); long bufferSize = ConfigExtension.getTransactionCacheBlockSize(stormConf) != null ? ConfigExtension.getTransactionCacheBlockSize(stormConf) : (1 * SizeUnit.GB); rocksDbOpt.setWriteBufferSize(bufferSize); int maxBufferNum = ConfigExtension.getTransactionMaxCacheBlockNum(stormConf) != null ? ConfigExtension.getTransactionMaxCacheBlockNum(stormConf) : 3; rocksDbOpt.setMaxWriteBufferNumber(maxBufferNum); // Config for log of RocksDb rocksDbOpt.setMaxLogFileSize(1073741824); // 1G rocksDbOpt.setKeepLogFileNum(1); rocksDbOpt.setInfoLogLevel(InfoLogLevel.WARN_LEVEL); try { Map<Object, Object> conf = new HashMap<Object, Object>(); conf.put(ROCKSDB_ROOT_DIR, cacheDir); conf.put(ROCKSDB_RESET, true); initDir(conf); initDb(null, rocksDbOpt); } catch (Exception e) { throw new RuntimeException(e); } kryo = new Kryo(); output = new Output(200, 2000000000); input = new Input(1); LOG.info("Finished rocksDb cache init: maxFlushSize={}, bufferSize={}, maxBufferNum={}", maxFlushSize, bufferSize, maxBufferNum); }
public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) { this.tableDesc = tableDesc; this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns); this.dbPath = dbPath; this.writeBatchSize = 500; this.options = new Options(); options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3) .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION) .setCompactionStyle(CompactionStyle.UNIVERSAL); }
@Override protected void init() throws FailStoreException { try { options = new Options(); options.setCreateIfMissing(true) .setWriteBufferSize(8 * SizeUnit.KB) .setMaxWriteBufferNumber(3) .setMaxBackgroundCompactions(10) .setCompressionType(CompressionType.SNAPPY_COMPRESSION) .setCompactionStyle(CompactionStyle.UNIVERSAL); Filter bloomFilter = new BloomFilter(10); BlockBasedTableConfig tableConfig = new BlockBasedTableConfig(); tableConfig.setBlockCacheSize(64 * SizeUnit.KB) .setFilter(bloomFilter) .setCacheNumShardBits(6) .setBlockSizeDeviation(5) .setBlockRestartInterval(10) .setCacheIndexAndFilterBlocks(true) .setHashIndexAllowCollision(false) .setBlockCacheCompressedSize(64 * SizeUnit.KB) .setBlockCacheCompressedNumShardBits(10); options.setTableFormatConfig(tableConfig); } catch (Exception e) { throw new FailStoreException(e); } }
@Override protected void init() throws FailStoreException { try { options = new Options(); options.setCreateIfMissing(true) .setWriteBufferSize(8 * SizeUnit.KB) .setMaxWriteBufferNumber(3) .setMaxBackgroundCompactions(10) .setCompressionType(CompressionType.SNAPPY_COMPRESSION) .setCompactionStyle(CompactionStyle.UNIVERSAL); Filter bloomFilter = new BloomFilter(10); BlockBasedTableConfig tableConfig = new BlockBasedTableConfig(); tableConfig.setBlockCacheSize(64 * SizeUnit.KB) .setFilter(bloomFilter) .setCacheNumShardBits(6) .setBlockSizeDeviation(5) .setBlockRestartInterval(10) .setCacheIndexAndFilterBlocks(true) .setHashIndexAllowCollision(false) .setBlockCacheCompressedSize(64 * SizeUnit.KB) .setBlockCacheCompressedNumShardBits(10); options.setTableFormatConfig(tableConfig); } catch (Exception e) { throw new FailStoreException(e); } }
currentOptions.setWriteBufferSize(64 * SizeUnit.MB);
@Override public Options setWriteBufferSize(final long writeBufferSize) { assert(isOwningHandle()); setWriteBufferSize(nativeHandle_, writeBufferSize); return this; }
.setWriteBufferSize(16 * 1024 * 1024)// .setMaxWriteBufferNumber(8)// .setMaxBackgroundCompactions(2)//
long bufferSize = ConfigExtension.getTransactionCacheBlockSize(stormConf) != null ? ConfigExtension.getTransactionCacheBlockSize(stormConf) : (1 * SizeUnit.GB); rocksDbOpt.setWriteBufferSize(bufferSize); int maxBufferNum = ConfigExtension.getTransactionMaxCacheBlockNum(stormConf) != null ? ConfigExtension.getTransactionMaxCacheBlockNum(stormConf) : 3;
private Options createDatabaseOptions() { BlockBasedTableConfig tableFormatConfig = new BlockBasedTableConfig() .setBlockSize(cacheBlockSizeKB * 1024L) .setBlockCacheSize(readCacheSizeMB * 1024L * 1024L) .setCacheIndexAndFilterBlocks(true); return new Options() .setCreateIfMissing(true) .setDbLogDir(Paths.get(this.dbDir, DB_LOG_DIR).toString()) .setWalDir(Paths.get(this.dbDir, DB_WRITE_AHEAD_LOG_DIR).toString()) .setWalTtlSeconds(0) .setWalSizeLimitMB(MAX_WRITE_AHEAD_LOG_SIZE_MB) .setWriteBufferSize(writeBufferSizeMB * 1024L * 1024L) .setMaxWriteBufferNumber(MAX_WRITE_BUFFER_NUMBER) .setMinWriteBufferNumberToMerge(MIN_WRITE_BUFFER_NUMBER_TO_MERGE) .setTableFormatConfig(tableFormatConfig) .setOptimizeFiltersForHits(true) .setUseDirectReads(true); }
public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) { this.tableDesc = tableDesc; this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns); this.dbPath = dbPath; this.writeBatchSize = 500; this.options = new Options(); options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3) .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION) .setCompactionStyle(CompactionStyle.UNIVERSAL); }
options.setWriteBufferSize(WRITE_BUFFER_SIZE); options.setCompressionType(COMPRESSION_TYPE); options.setCompactionStyle(COMPACTION_STYLE);
/** * Builds RocksDb {@link Options}. * * @param maxBackgroundThreads * @param levelZeloFileNumCompactionTrigger * @param writeBufferSize * @param targetFileSizeBase * @return */ public static Options buildOptions(int maxBackgroundThreads, int levelZeloFileNumCompactionTrigger, long writeBufferSize, long targetFileSizeBase) { Options rocksOptions = new Options(); rocksOptions.setCreateIfMissing(true).getEnv().setBackgroundThreads(1, Env.FLUSH_POOL) .setBackgroundThreads(maxBackgroundThreads, Env.COMPACTION_POOL); rocksOptions.setMaxBackgroundFlushes(1).setMaxBackgroundCompactions(maxBackgroundThreads); rocksOptions.setWriteBufferSize(writeBufferSize).setMinWriteBufferNumberToMerge(2) .setLevelZeroFileNumCompactionTrigger(levelZeloFileNumCompactionTrigger) .setTargetFileSizeBase(targetFileSizeBase); rocksOptions.setMemTableConfig(new SkipListMemTableConfig()); // rocksOptions.setMemTableConfig(new HashSkipListMemTableConfig()); // rocksOptions.setMemTableConfig(new HashLinkedListMemTableConfig()); return rocksOptions; }
Long writeBufSize = storeConfig.getLong("container.write.buffer.size.bytes", 32 * 1024 * 1024); options.setWriteBufferSize((int) (writeBufSize / numTasksForContainer));
Long writeBufSize = storeConfig.getLong("container.write.buffer.size.bytes", 32 * 1024 * 1024); options.setWriteBufferSize((int) (writeBufSize / numTasksForContainer));
options.setWriteBufferSize(bufferSize);
Long writeBufSize = storeConfig.getLong("container.write.buffer.size.bytes", 32 * 1024 * 1024); options.setWriteBufferSize((int) (writeBufSize / numTasksForContainer));
@Override protected void init() throws FailStoreException { try { options = new Options(); options.setCreateIfMissing(true) .setWriteBufferSize(8 * SizeUnit.KB) .setMaxWriteBufferNumber(3) .setMaxBackgroundCompactions(10) .setCompressionType(CompressionType.SNAPPY_COMPRESSION) .setCompactionStyle(CompactionStyle.UNIVERSAL); Filter bloomFilter = new BloomFilter(10); BlockBasedTableConfig tableConfig = new BlockBasedTableConfig(); tableConfig.setBlockCacheSize(64 * SizeUnit.KB) .setFilter(bloomFilter) .setCacheNumShardBits(6) .setBlockSizeDeviation(5) .setBlockRestartInterval(10) .setCacheIndexAndFilterBlocks(true) .setHashIndexAllowCollision(false) .setBlockCacheCompressedSize(64 * SizeUnit.KB) .setBlockCacheCompressedNumShardBits(10); options.setTableFormatConfig(tableConfig); } catch (Exception e) { throw new FailStoreException(e); } }
options.setCompressionType(CompressionType.LZ4_COMPRESSION); options.setWriteBufferSize(writeBufferSizeMB * 1024 * 1024); options.setMaxWriteBufferNumber(4); if (numLevels > 0) {