@Override public void put(String key, Object value) throws FailStoreException { String valueString = JSON.toJSONString(value); WriteOptions writeOpts = new WriteOptions(); try { writeOpts.setSync(true); writeOpts.setDisableWAL(true); db.put(writeOpts, key.getBytes("UTF-8"), valueString.getBytes("UTF-8")); } catch (Exception e) { throw new FailStoreException(e); } finally { writeOpts.dispose(); } }
@Override public void put(String key, Object value) throws FailStoreException { String valueString = JSON.toJSONString(value); WriteOptions writeOpts = new WriteOptions(); try { writeOpts.setSync(true); writeOpts.setDisableWAL(true); db.put(writeOpts, key.getBytes("UTF-8"), valueString.getBytes("UTF-8")); } catch (Exception e) { throw new FailStoreException(e); } finally { writeOpts.dispose(); } }
/** * Creates the RocksDB WriteOptions to use. Since we use RocksDB as an in-process cache with disk spillover, * we do not care about the data being persisted to disk for recovery purposes. As such: * * Write-Ahead-Log is disabled (2.8x performance improvement) * * Sync is disabled - does not wait for a disk flush before returning from the write call (50x or more improvement). */ private WriteOptions createWriteOptions() { return new WriteOptions() .setDisableWAL(true) .setSync(false); }
public RocksdbMap() { dbDir = Files.createTempDir(); try { this.db = RocksDB.open(dbDir.getAbsolutePath()); writeOptions = new WriteOptions(); writeOptions.setDisableWAL(true); writeOptions.setSync(false); } catch (RocksDBException e) { throw new RuntimeException(e); } }
/** * Builds RocksDb {@link WriteOptions}. * * @param sync * As of RocksDB v3.10.1, no data lost if process crashes even if sync==false. Data * lost only when server/OS crashes. So it's relatively safe to set sync=false for * fast write * @param disableWAL * @return */ public static WriteOptions buildWriteOptions(boolean sync, boolean disableWAL) { WriteOptions writeOptions = new WriteOptions(); writeOptions.setSync(sync).setDisableWAL(disableWAL); return writeOptions; }
private RocksDB db() { if (_db == null) { writeOptions = new WriteOptions(); writeOptions.setDisableWAL(true); writeOptions.setSync(false); readOptions = new ReadOptions(); readOptions.setFillCache(true).setVerifyChecksums(false); _db = _dbSupplier.get(); } return _db; }
@Override public void put(String key, Object value) throws FailStoreException { String valueString = JSON.toJSONString(value); WriteOptions writeOpts = new WriteOptions(); try { writeOpts.setSync(true); writeOpts.setDisableWAL(true); db.put(writeOpts, key.getBytes("UTF-8"), valueString.getBytes("UTF-8")); } catch (Exception e) { throw new FailStoreException(e); } finally { writeOpts.dispose(); } }
private RocksDB db() { if (_db == null) { writeOptions = new WriteOptions(); writeOptions.setDisableWAL(true); writeOptions.setSync(false); readOptions = new ReadOptions(); readOptions.setFillCache(false).setVerifyChecksums(false); _db = _dbSupplier.get(); } return _db; }
@Override public void removeConflicts(@Nullable String txId, Iterable<String> paths) { Optional<RocksDBReference> dbRefOpt = getDb(txId); if (!dbRefOpt.isPresent()) { return; } try (RocksDBReference dbRef = dbRefOpt.get(); WriteOptions writeOptions = new WriteOptions(); WriteBatch batch = new WriteBatch()) { writeOptions.setSync(true); for (String path : paths) { batch.remove(key(path)); } dbRef.db().write(writeOptions, batch); } catch (RocksDBException e) { throw new RuntimeException(e); } }
@Override public void removeConflicts(@Nullable String txId, Iterable<String> paths) { Optional<RocksDBReference> dbRefOpt = getDb(txId); if (!dbRefOpt.isPresent()) { return; } try (RocksDBReference dbRef = dbRefOpt.get(); WriteOptions writeOptions = new WriteOptions(); WriteBatch batch = new WriteBatch()) { writeOptions.setSync(true); for (String path : paths) { batch.remove(key(path)); } dbRef.db().write(writeOptions, batch); } catch (RocksDBException e) { propagate(e); } }
@Override public boolean put(ObjectId commitId, ImmutableList<ObjectId> parentIds) { try (WriteBatchWithIndex batch = new WriteBatchWithIndex(); // RocksDBReference dbRef = dbhandle.getReference(); WriteOptions wo = new WriteOptions()) { wo.setSync(true); boolean updated = put(dbRef, commitId, parentIds, batch); dbRef.db().write(wo, batch); return updated; } catch (RocksDBException e) { throw new RuntimeException(e); } }
@Override public void putMetadataForTables(Map<TableReference, byte[]> tableRefToMetadata) { try (Disposer d = new Disposer(); ColumnFamily metadataTable = columnFamilies.get(METADATA_TABLE_NAME.getQualifiedName())) { WriteOptions options = d.register(new WriteOptions().setSync(true)); WriteBatch batch = d.register(new WriteBatch()); for (Entry<TableReference, byte[]> entry : tableRefToMetadata.entrySet()) { batch.put(metadataTable.getHandle(), entry.getKey().getQualifiedName().getBytes(Charsets.UTF_8), entry.getValue()); } getDb().write(options, batch); } catch (RocksDBException e) { throw Throwables.propagate(e); } }
@Override public boolean put(ObjectId commitId, ImmutableList<ObjectId> parentIds) { try (WriteBatchWithIndex batch = new WriteBatchWithIndex(); // RocksDBReference dbRef = dbhandle.getReference(); WriteOptions wo = new WriteOptions()) { wo.setSync(true); boolean updated = put(dbRef, commitId, parentIds, batch); dbRef.db().write(wo, batch); return updated; } catch (Exception e) { throw Throwables.propagate(e); } }
@Override public void putMetadataForTable(TableReference tableRef, byte[] metadata) { try (Disposer d = new Disposer(); ColumnFamily metadataTable = columnFamilies.get(METADATA_TABLE_NAME.getQualifiedName())) { WriteOptions options = d.register(new WriteOptions().setSync(true)); getDb().put(metadataTable.getHandle(), options, tableRef.getQualifiedName().getBytes(Charsets.UTF_8), metadata); } catch (RocksDBException e) { throw Throwables.propagate(e); } }
@Override public void put(TableReference tableRef, Map<Cell, byte[]> values, long timestamp) { try (Disposer d = new Disposer(); ColumnFamily table = columnFamilies.get(tableRef.getQualifiedName())) { WriteOptions options = d.register(new WriteOptions().setSync(writeOptions.fsyncPut())); WriteBatch batch = d.register(new WriteBatch()); for (Entry<Cell, byte[]> entry : values.entrySet()) { byte[] key = RocksDbKeyValueServices.getKey(entry.getKey(), timestamp); batch.put(table.getHandle(), key, entry.getValue()); } getDb().write(options, batch); } catch (RocksDBException e) { throw Throwables.propagate(e); } }
@Override public void delete(TableReference tableRef, Multimap<Cell, Long> keys) { try (Disposer d = new Disposer(); ColumnFamily table = columnFamilies.get(tableRef.getQualifiedName())) { WriteOptions options = d.register(new WriteOptions().setSync(writeOptions.fsyncPut())); WriteBatch batch = d.register(new WriteBatch()); for (Entry<Cell, Long> entry : keys.entries()) { byte[] key = RocksDbKeyValueServices.getKey(entry.getKey(), entry.getValue()); batch.remove(table.getHandle(), key); } getDb().write(options, batch); } catch (RocksDBException e) { throw Throwables.propagate(e); } }
@Override public void putWithTimestamps(TableReference tableRef, Multimap<Cell, Value> cellValues) { try (Disposer d = new Disposer(); ColumnFamily table = columnFamilies.get(tableRef.getQualifiedName())) { WriteOptions options = d.register(new WriteOptions().setSync(writeOptions.fsyncPut())); WriteBatch batch = d.register(new WriteBatch()); for (Entry<Cell, Value> entry : cellValues.entries()) { Value value = entry.getValue(); byte[] key = RocksDbKeyValueServices.getKey(entry.getKey(), value.getTimestamp()); batch.put(table.getHandle(), key, value.getContents()); } getDb().write(options, batch); } catch (RocksDBException e) { throw Throwables.propagate(e); } }
@Override public void addConflicts(@Nullable String txId, Iterable<Conflict> conflicts) { try (RocksDBReference dbRef = getOrCreateDb(txId)) { ConflictSerializer serializer = new ConflictSerializer(); try (WriteBatch batch = new WriteBatch()) { for (Conflict c : conflicts) { byte[] key = key(c.getPath()); byte[] value = serializer.write(c); batch.put(key, value); } try (WriteOptions writeOptions = new WriteOptions()) { writeOptions.setSync(true); dbRef.db().write(writeOptions, batch); } } catch (RocksDBException | IOException e) { throw new RuntimeException(e); } } }
@Override public void addConflicts(@Nullable String txId, Iterable<Conflict> conflicts) { try (RocksDBReference dbRef = getOrCreateDb(txId)) { ConflictSerializer serializer = new ConflictSerializer(); try (WriteBatch batch = new WriteBatch()) { for (Conflict c : conflicts) { byte[] key = key(c.getPath()); byte[] value = serializer.write(c); batch.put(key, value); } try (WriteOptions writeOptions = new WriteOptions()) { writeOptions.setSync(true); dbRef.db().write(writeOptions, batch); } } catch (Exception e) { propagate(e); } } }
@Override public void truncate() { try (RocksDBReference dbRef = dbhandle.getReference()) { try (RocksIterator it = dbRef.db().newIterator()) { it.seekToFirst(); try (WriteOptions wo = new WriteOptions(); // WriteBatch batch = new WriteBatch()) { wo.setSync(true); while (it.isValid()) { batch.remove(it.key()); it.next(); } dbRef.db().write(wo, batch); } } catch (RocksDBException e) { throw propagate(e); } } }