/** * <p>Set appropriate parameters for bulk loading. * The reason that this is a function that returns "this" instead of a * constructor is to enable chaining of multiple similar calls in the future. * </p> * * <p>All data will be in level 0 without any automatic compaction. * It's recommended to manually call CompactRange(NULL, NULL) before reading * from the database, because otherwise the read can be very slow.</p> * * @return the instance of the current Options. */ public Options prepareForBulkLoad() { prepareForBulkLoad(nativeHandle_); return this; }
@Override public SuffixDataStore createSuffixDataStore(int id) { RocksDB.loadLibrary(); try (Options options = new Options().setCreateIfMissing(true).prepareForBulkLoad()) { Files.createDirectories(dbPath); RocksDB rocksDB = RocksDB.open(options, dbPath.resolve(getSuffixesName(id)).toString()); rocksDBS.add(rocksDB); return new RocksDbSuffixDataStore(rocksDB); } catch (RocksDBException | IOException e) { throw new RuntimeException(e); } }
@Override public KnownWordsDataStore createKnownWordsDataStore(int id) { RocksDB.loadLibrary(); try (Options options = new Options().setCreateIfMissing(true).prepareForBulkLoad()) { Files.createDirectories(dbPath); RocksDB rocksDB = RocksDB.open(options, dbPath.resolve(getWordsName(id)).toString()); rocksDBS.add(rocksDB); RocksDB candidatesDB = RocksDB.open(options, dbPath.resolve(getCandidatesName(id)).toString()); rocksDBS.add(candidatesDB); return new RocksDbKnownWordsDataStore(rocksDB, candidatesDB); } catch (RocksDBException | IOException e) { throw new RuntimeException(e); } }
public RocksDBSenseVectors(Path path, boolean forWriting) { RocksDB.loadLibrary(); if (forWriting) { try (Options options = new Options().setCreateIfMissing(true).prepareForBulkLoad()) { rocksDB = RocksDB.open(options, path.toString()); } catch (RocksDBException e) { throw new RuntimeException(e); } } else { try { rocksDB = RocksDB.openReadOnly(path.toString()); } catch (RocksDBException e) { throw new RuntimeException(e); } } }
@Override void setOutputPath(Path outputPath) { try { Files.createDirectories(outputPath); } catch (IOException e) { throw new RuntimeException(e); } try (Options options = new Options().setCreateIfMissing(true).prepareForBulkLoad()){ try { words = new RocksDbTermIndexBuilder( RocksDB.open(options, outputPath.resolve("wordsTerms").toString()), RocksDB.open(options, outputPath.resolve("wordsIndices").toString())); terms = new RocksDbTermIndexBuilder( RocksDB.open(options, outputPath.resolve("termsTerms").toString()), RocksDB.open(options, outputPath.resolve("termsIndices").toString())); norms = new RocksDbTermIndexBuilder( RocksDB.open(options, outputPath.resolve("normsTerms").toString()), RocksDB.open(options, outputPath.resolve("normsIndices").toString())); } catch (RocksDBException e) { throw new RuntimeException(e); } } }
public synchronized RocksDBDataIndexTable getDataIndexTable( final String tableName, final short adapterId) { if (indexWriteOptions == null) { RocksDB.loadLibrary(); final int cores = Runtime.getRuntime().availableProcessors(); indexWriteOptions = new Options().setCreateIfMissing(true).prepareForBulkLoad().setIncreaseParallelism(cores); indexReadOptions = new Options().setIncreaseParallelism(cores); } final String directory = subDirectory + "/" + tableName; return dataIndexTableCache.get( (DataIndexCacheKey) keyCache.get(directory, d -> new DataIndexCacheKey(d, adapterId))); }
public synchronized RocksDBIndexTable getIndexTable( final String tableName, final short adapterId, final byte[] partition, final boolean requiresTimestamp) { if (indexWriteOptions == null) { RocksDB.loadLibrary(); final int cores = Runtime.getRuntime().availableProcessors(); indexWriteOptions = new Options().setCreateIfMissing(true).prepareForBulkLoad().setIncreaseParallelism(cores); indexReadOptions = new Options().setIncreaseParallelism(cores); } final String directory = subDirectory + "/" + tableName; return indexTableCache.get( (IndexCacheKey) keyCache.get( directory, d -> new IndexCacheKey(d, adapterId, partition, requiresTimestamp))); }
options.prepareForBulkLoad();