@Test(expected = PermanentLockingException.class) public void expectedValueMismatchCausesMutateFailure() throws BackendException { store[0].acquireLock(k, c1, v1, tx[0][0]); store[0].mutate(k, Collections.singletonList(StaticArrayEntry.of(c1, v1)), NO_DELETIONS, tx[0][0]); }
@Override public void close() throws BackendException { store.close(); } }
@Override public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) throws BackendException { return store.getKeys(query,txh); }
private void runWithExceptions() throws BackendException { StaticBuffer lockKey = serializer.toLockKey(target.getKey(), target.getColumn()); List<Entry> locks = store.getSlice(new KeySliceQuery(lockKey, LOCK_COL_START, LOCK_COL_END), tx); // TODO reduce LOCK_COL_END based on cutoff ImmutableList.Builder<StaticBuffer> b = ImmutableList.builder(); for (Entry lc : locks) { TimestampRid tr = serializer.fromLockColumn(lc.getColumn(), times); if (tr.getTimestamp().isBefore(cutoff)) { log.info("Deleting expired lock on {} by rid {} with timestamp {} (before or at cutoff {})", target, tr.getRid(), tr.getTimestamp(), cutoff); b.add(lc.getColumn()); } else { log.debug("Ignoring lock on {} by rid {} with timestamp {} (timestamp is after cutoff {})", target, tr.getRid(), tr.getTimestamp(), cutoff); } } List<StaticBuffer> deletions = b.build(); if (!deletions.isEmpty()) { store.mutate(lockKey, ImmutableList.of(), deletions, tx); log.info("Deleted {} expired locks (before or at cutoff {})", deletions.size(), cutoff); } }
@Override public EntryList getSlice(KeySliceQuery query, StoreTransaction txh) throws BackendException { getSliceCounter.incrementAndGet(); return store.getSlice(query,txh); }
@Override public void mutate(StaticBuffer key, List<Entry> additions, List<StaticBuffer> deletions, StoreTransaction txh) throws BackendException { store.mutate(key,additions,deletions,txh); }
@Test public void testMutateWithLockUsesConsistentTx() throws BackendException { final ImmutableList<Entry> adds = ImmutableList.of(StaticArrayEntry.of(DATA_COL, DATA_VAL)); final ImmutableList<StaticBuffer> deletions = ImmutableList.of(); final KeyColumn kc = new KeyColumn(LOCK_KEY, LOCK_COL); // 1. Acquire a lock backingLocker.writeLock(kc, consistentTx); // 2. Run a mutation // N.B. mutation coordinates do not overlap with the lock, but consistentTx should be used anyway // 2.1. Check locks & expected values before mutating data backingLocker.checkLocks(consistentTx); StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn()); KeySliceQuery expectedValueQuery = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf); expect(backingStore.getSlice(expectedValueQuery, consistentTx)) // expected value read must use strong consistency .andReturn(StaticArrayEntryList.of(StaticArrayEntry.of(LOCK_COL, LOCK_VAL))); // 2.2. Mutate data backingStore.mutate(DATA_KEY, adds, deletions, consistentTx); // writes by txs with locks must use strong consistency ctrl.replay(); // 1. Lock acquisition expectStore.acquireLock(LOCK_KEY, LOCK_COL, LOCK_VAL, expectTx); // 2. Mutate expectStore.mutate(DATA_KEY, adds, deletions, expectTx); }
/** * Retrieves the value for the specified column and key under the given transaction * from the store if such exists, otherwise returns NULL * * @param store Store * @param key Key * @param column Column * @param txh Transaction * @return Value for key and column or NULL if such does not exist */ public static StaticBuffer get(KeyColumnValueStore store, StaticBuffer key, StaticBuffer column, StoreTransaction txh) throws BackendException { KeySliceQuery query = new KeySliceQuery(key, column, BufferUtil.nextBiggerBuffer(column)).setLimit(2); List<Entry> result = store.getSlice(query, txh); if (result.size() > 1) log.warn("GET query returned more than 1 result: store {} | key {} | column {}", store.getName(), key, column); if (result.isEmpty()) return null; else return result.get(0).getValueAs(StaticBuffer.STATIC_FACTORY); }
@Override public void acquireLock(StaticBuffer key, StaticBuffer column, StaticBuffer expectedValue, StoreTransaction txh) throws BackendException { store.acquireLock(key,column,expectedValue,txh); }
@Override public String getName() { return backend.getName(); }
@Test public void testMutateManyWithLockUsesConsistentTx() throws BackendException { final ImmutableList<Entry> adds = ImmutableList.of(StaticArrayEntry.of(DATA_COL, DATA_VAL)); final ImmutableList<StaticBuffer> deletions = ImmutableList.of(); Map<String, Map<StaticBuffer, KCVMutation>> mutations = ImmutableMap.of(STORE_NAME, ImmutableMap.of(DATA_KEY, new KCVMutation(adds, deletions))); final KeyColumn kc = new KeyColumn(LOCK_KEY, LOCK_COL); // Acquire a lock backingLocker.writeLock(kc, consistentTx); // 2. Run mutateMany // 2.1. Check locks & expected values before mutating data backingLocker.checkLocks(consistentTx); StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn()); KeySliceQuery expectedValueQuery = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf); expect(backingStore.getSlice(expectedValueQuery, consistentTx)) // expected value read must use strong consistency .andReturn(StaticArrayEntryList.of(StaticArrayEntry.of(LOCK_COL, LOCK_VAL))); // 2.2. Run mutateMany on backing manager to modify data backingManager.mutateMany(mutations, consistentTx); // writes by txs with locks must use strong consistency ctrl.replay(); // Lock acquisition expectStore.acquireLock(LOCK_KEY, LOCK_COL, LOCK_VAL, expectTx); // Mutate expectManager.mutateMany(mutations, expectTx); }
public void testDataSequential() throws Exception { loadData(200000,2); close(); KeyColumnValueStoreManager manager = openStorageManager(); KeyColumnValueStore store = manager.openDatabase(Backend.EDGESTORE_NAME); SliceQuery query = new SliceQuery(BufferUtil.zeroBuffer(8),BufferUtil.oneBuffer(8)); query.setLimit(2); Stopwatch watch = Stopwatch.createStarted(); StoreTransaction txh = manager.beginTransaction(StandardBaseTransactionConfig.of(TimestampProviders.MILLI)); KeyIterator iterator = store.getKeys(query,txh); int numV = 0; while(iterator.hasNext()) { iterator.next(); RecordIterator<Entry> entries = iterator.getEntries(); assertEquals(2, Iterators.size(entries)); numV++; } iterator.close(); txh.commit(); System.out.println("Time taken: " + watch.elapsed(TimeUnit.MILLISECONDS)); System.out.println("Num Vertices: " + numV); store.close(); manager.close(); }
StaticBuffer end = KeyColumnValueStoreUtil.stringToByteBuffer("d"); EntryList results = storeWithTTL.getSlice(new KeySliceQuery(key, new SliceQuery(start, end)), tx); Assert.assertEquals(3, results.size()); tx = startTx(); results = storeWithTTL.getSlice(new KeySliceQuery(key, new SliceQuery(start, end)), tx); Assert.assertEquals(0, results.size()); // should be empty if TTL was applied properly storeWithTTL.close();
entries.add(StaticArrayEntry.of(col, col)); store.mutate(key, entries, KeyColumnValueStore.NO_DELETIONS, tx); tx.commit(); StaticBuffer columnEnd = KeyColumnValueStoreUtil.longToByteBuffer(cols); List<Entry> result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(cols), tx); Assert.assertEquals(cols, result.size()); result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(cols + 10), tx); Assert.assertEquals(cols, result.size()); Assert.assertEquals(entries, result); result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(cols - 1), tx); Assert.assertEquals(cols - 1, result.size()); entries.remove(entries.size() - 1); Assert.assertEquals(entries, result); result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(1), tx); Assert.assertEquals(1, result.size()); final List<Entry> firstEntrySingleton = Collections.singletonList(entries.get(0));
@Override public Map<StaticBuffer, EntryList> getSlice(List<StaticBuffer> keys, SliceQuery query, StoreTransaction txh) throws BackendException { getSliceCounter.incrementAndGet(); return store.getSlice(keys,query,txh); }
@Override public void mutate(final StaticBuffer key, final List<Entry> additions, final List<StaticBuffer> deletions, final StoreTransaction txh) throws BackendException { runWithMetrics(txh, metricsStoreName, M_MUTATE, (StorageCallable<Void>) () -> { backend.mutate(key, additions, deletions, txh); return null; }); }
@Override public void acquireLock(StaticBuffer key, StaticBuffer column, StaticBuffer expectedValue, StoreTransaction txh) throws BackendException { store.acquireLock(key,column,expectedValue,unwrapTx(txh)); }
@Override public String getName() { return store.getName(); }
private void tryLocks(KeyColumnValueStore s1, StoreTransaction tx1, KeyColumnValueStore s2, StoreTransaction tx2, boolean detectLocally) throws BackendException, InterruptedException { s1.acquireLock(k, k, null, tx1); // Require local lock contention, if requested by our caller // Remote lock contention is checked by separate cases if (detectLocally) { try { s2.acquireLock(k, k, null, tx2); Assert.fail("Expected lock contention between transactions did not occur"); } catch (BackendException e) { Assert.assertTrue(e instanceof PermanentLockingException || e instanceof TemporaryLockingException); } } // Let the original lock expire Thread.sleep(EXPIRE_MS + 100L); // This should succeed now that the original lock is expired s2.acquireLock(k, k, null, tx2); // Mutate to check for remote contention s2.mutate(k, Collections.singletonList(StaticArrayEntry.of(c2, v2)), NO_DELETIONS, tx2); }
store.mutate(key, additions, KeyColumnValueStore.NO_DELETIONS, tx); tx.commit(); StaticBuffer columnEnd = KeyColumnValueStoreUtil.longToByteBuffer(ttls.length); List<Entry> result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(ttls.length), tx); Assert.assertEquals(ttls.length, result.size()); store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(ttls.length), tx); Assert.assertEquals(ttls.length - 1, result.size()); store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(ttls.length), tx); Assert.assertEquals(ttls.length - 1, result.size()); tx.rollback(); result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(ttls.length), tx); Assert.assertEquals(ttls.length - 2, result.size()); tx.rollback(); result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd).setLimit(ttls.length), tx); Assert.assertEquals(ttls.length - 2, result.size());