@Override public KeySliceQuery updateLimit(int newLimit) { return new KeySliceQuery(key,this).setLimit(newLimit); }
@Override public Map<StaticBuffer,EntryList> getSlice(List<StaticBuffer> keys, SliceQuery query, StoreTransaction txh) throws BackendException { Map<StaticBuffer,EntryList> result = Maps.newHashMap(); for (StaticBuffer key : keys) result.put(key,getSlice(new KeySliceQuery(key,query),txh)); return result; }
@Override public boolean hasNext() { ensureOpen(); if (null != nextRow) return true; while (rows.hasNext()) { nextRow = rows.next(); List<Entry> entries = nextRow.getValue().getSlice(new KeySliceQuery(nextRow.getKey(), columnSlice), transaction); if (null != entries && 0 < entries.size()) break; } return null != nextRow; }
@Override public Map<StaticBuffer,EntryList> getSlice(List<StaticBuffer> keys, SliceQuery query, StoreTransaction txh) throws BackendException { final List<KVQuery> queries = new ArrayList<>(keys.size()); for (StaticBuffer key : keys) { queries.add(convertQuery(new KeySliceQuery(key, query))); } final Map<KVQuery,RecordIterator<KeyValueEntry>> results = store.getSlices(queries,txh); final Map<StaticBuffer,EntryList> convertedResults = new HashMap<>(keys.size()); assert queries.size()==keys.size(); for (int i = 0; i < queries.size(); i++) { convertedResults.put(keys.get(i),convert(results.get(queries.get(i)))); } return convertedResults; }
public EntryList edgeQuery(long vid, SliceQuery query, BackendTransaction tx) { Preconditions.checkArgument(vid > 0); return tx.edgeStoreQuery(new KeySliceQuery(idManager.getKey(vid), query)); }
public MultiKeySliceQuery getQuery(final CompositeIndexType index, List<Object[]> values) { final List<KeySliceQuery> ksqs = new ArrayList<>(values.size()); for (final Object[] value : values) { ksqs.add(new KeySliceQuery(getIndexKey(index,value), BufferUtil.zeroBuffer(1), BufferUtil.oneBuffer(1))); } return new MultiKeySliceQuery(ksqs); }
public static KeySliceQuery getQuery(int key, int startCol, int endCol) { return new KeySliceQuery(BufferUtil.getIntBuffer(key),getQuery(startCol, endCol)); }
ksqs[i] = new KeySliceQuery(key,query); EntryList result = null; if (!isExpired(ksqs[i])) result = cache.getIfPresent(ksqs[i]);
@Override public RecordIterator<Entry> getEntries() { ensureOpen(); if (columnSlice == null) throw new IllegalStateException("getEntries() requires SliceQuery to be set."); final KeySliceQuery keySlice = new KeySliceQuery(currentRow.getKey(), columnSlice); return new RecordIterator<Entry>() { private final Iterator<Entry> items = currentRow.getValue().getSlice(keySlice, transaction).iterator(); @Override public boolean hasNext() { ensureOpen(); return items.hasNext(); } @Override public Entry next() { ensureOpen(); return items.next(); } @Override public void close() { isClosed = true; } @Override public void remove() { throw new UnsupportedOperationException("Column removal not supported"); } }; }
/** * Retrieves the value for the specified column and key under the given transaction * from the store if such exists, otherwise returns NULL * * @param store Store * @param key Key * @param column Column * @param txh Transaction * @return Value for key and column or NULL if such does not exist */ public static StaticBuffer get(KeyColumnValueStore store, StaticBuffer key, StaticBuffer column, StoreTransaction txh) throws BackendException { KeySliceQuery query = new KeySliceQuery(key, column, BufferUtil.nextBiggerBuffer(column)).setLimit(2); List<Entry> result = store.getSlice(query, txh); if (result.size() > 1) log.warn("GET query returned more than 1 result: store {} | key {} | column {}", store.getName(), key, column); if (result.isEmpty()) return null; else return result.get(0).getValueAs(StaticBuffer.STATIC_FACTORY); }
public static boolean containsKey(KeyColumnValueStore store, StaticBuffer key, int maxColumnLength, StoreTransaction txh) throws BackendException { final StaticBuffer end; if (maxColumnLength>32) { end = BufferUtil.oneBuffer(maxColumnLength); } else { end = END; } return !store.getSlice(new KeySliceQuery(key, START, end).setLimit(1),txh).isEmpty(); }
private Map<String,Object> toMap() { Map<String,Object> entries = Maps.newHashMap(); List<Entry> result = BackendOperation.execute(new BackendOperation.Transactional<List<Entry>>() { @Override public List<Entry> call(StoreTransaction txh) throws BackendException { return store.getSlice(new KeySliceQuery(rowKey, BufferUtil.zeroBuffer(1), BufferUtil.oneBuffer(128)),txh); } @Override public String toString() { return "setConfiguration"; } },txProvider, times, maxOperationWaitTime); for (Entry entry : result) { String key = staticBuffer2String(entry.getColumnAs(StaticBuffer.STATIC_FACTORY)); Object value = staticBuffer2Object(entry.getValueAs(StaticBuffer.STATIC_FACTORY), Object.class); entries.put(key,value); } return entries; }
private void runWithExceptions() throws BackendException { StaticBuffer lockKey = serializer.toLockKey(target.getKey(), target.getColumn()); List<Entry> locks = store.getSlice(new KeySliceQuery(lockKey, LOCK_COL_START, LOCK_COL_END), tx); // TODO reduce LOCK_COL_END based on cutoff ImmutableList.Builder<StaticBuffer> b = ImmutableList.builder(); for (Entry lc : locks) { TimestampRid tr = serializer.fromLockColumn(lc.getColumn(), times); if (tr.getTimestamp().isBefore(cutoff)) { log.info("Deleting expired lock on {} by rid {} with timestamp {} (before or at cutoff {})", target, tr.getRid(), tr.getTimestamp(), cutoff); b.add(lc.getColumn()); } else { log.debug("Ignoring lock on {} by rid {} with timestamp {} (timestamp is after cutoff {})", target, tr.getRid(), tr.getTimestamp(), cutoff); } } List<StaticBuffer> deletions = b.build(); if (!deletions.isEmpty()) { store.mutate(lockKey, ImmutableList.of(), deletions, tx); log.info("Deleted {} expired locks (before or at cutoff {})", deletions.size(), cutoff); } }
private long getCurrentID(final StaticBuffer partitionKey) throws BackendException { final List<Entry> blocks = BackendOperation.execute( (BackendOperation.Transactional<List<Entry>>) txh -> idStore.getSlice(new KeySliceQuery(partitionKey, LOWER_SLICE, UPPER_SLICE).setLimit(5), txh),this,times); if (blocks == null) throw new TemporaryBackendException("Could not read from storage"); long latest = BASE_ID; for (Entry e : blocks) { long counterVal = getBlockValue(e); if (latest < counterVal) { latest = counterVal; } } return latest; }
/** * Reads the configuration property for this StoreManager * * @param key Key identifying the configuration property * @return Value stored for the key or null if the configuration property has not (yet) been defined. */ @Override public <O> O get(final String key, final Class<O> dataType) { StaticBuffer column = string2StaticBuffer(key); final KeySliceQuery query = new KeySliceQuery(rowKey,column, BufferUtil.nextBiggerBuffer(column)); StaticBuffer result = BackendOperation.execute(new BackendOperation.Transactional<StaticBuffer>() { @Override public StaticBuffer call(StoreTransaction txh) throws BackendException { List<Entry> entries = store.getSlice(query,txh); if (entries.isEmpty()) return null; return entries.get(0).getValueAs(StaticBuffer.STATIC_FACTORY); } @Override public String toString() { return "getConfiguration"; } }, txProvider, times, maxOperationWaitTime); if (result==null) return null; return staticBuffer2Object(result, dataType); }
@Test public void testMutateManyWithLockUsesConsistentTx() throws BackendException { final ImmutableList<Entry> adds = ImmutableList.of(StaticArrayEntry.of(DATA_COL, DATA_VAL)); final ImmutableList<StaticBuffer> deletions = ImmutableList.of(); Map<String, Map<StaticBuffer, KCVMutation>> mutations = ImmutableMap.of(STORE_NAME, ImmutableMap.of(DATA_KEY, new KCVMutation(adds, deletions))); final KeyColumn kc = new KeyColumn(LOCK_KEY, LOCK_COL); // Acquire a lock backingLocker.writeLock(kc, consistentTx); // 2. Run mutateMany // 2.1. Check locks & expected values before mutating data backingLocker.checkLocks(consistentTx); StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn()); KeySliceQuery expectedValueQuery = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf); expect(backingStore.getSlice(expectedValueQuery, consistentTx)) // expected value read must use strong consistency .andReturn(StaticArrayEntryList.of(StaticArrayEntry.of(LOCK_COL, LOCK_VAL))); // 2.2. Run mutateMany on backing manager to modify data backingManager.mutateMany(mutations, consistentTx); // writes by txs with locks must use strong consistency ctrl.replay(); // Lock acquisition expectStore.acquireLock(LOCK_KEY, LOCK_COL, LOCK_VAL, expectTx); // Mutate expectManager.mutateMany(mutations, expectTx); }
public void checkSlice(String[][] values, Set<KeyColumn> removed, int key, int start, int end, int limit) throws BackendException { tx.rollback(); tx = startTx(); List<Entry> entries; if (limit <= 0) entries = store.getSlice(new KeySliceQuery(KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end)), tx); else entries = store.getSlice(new KeySliceQuery(KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end)).setLimit(limit), tx); int pos = 0; for (int i = start; i < end; i++) { if (removed.contains(new KeyColumn(key, i))) { log.debug("Skipping deleted ({},{})", key, i); continue; } if (limit <= 0 || pos < limit) { log.debug("Checking k={}[c_start={},c_end={}](limit={}): column index={}/pos={}", key, start, end, limit, i, pos); Assert.assertTrue(entries.size() > pos); Entry entry = entries.get(pos); int col = KeyValueStoreUtil.getID(entry.getColumn()); String str = KeyValueStoreUtil.getString(entry.getValueAs(StaticBuffer.STATIC_FACTORY)); Assert.assertEquals(i, col); Assert.assertEquals(values[key][i], str); } pos++; } Assert.assertNotNull(entries); if (limit > 0 && pos > limit) Assert.assertEquals(limit, entries.size()); else Assert.assertEquals(pos, entries.size()); }
@Test public void mutateManyWritesSameKeyOnMultipleCFs() throws BackendException { final long arbitraryLong = 42; //must be greater than 0 final StaticBuffer key = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong * arbitraryLong); final StaticBuffer val = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong * arbitraryLong * arbitraryLong); final StaticBuffer col = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong); final StaticBuffer nextCol = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong + 1); final StoreTransaction directTx = manager.beginTransaction(getTxConfig()); KCVMutation km = new KCVMutation( Lists.newArrayList(StaticArrayEntry.of(col, val)), Lists.newArrayList()); Map<StaticBuffer, KCVMutation> keyColumnAndValue = ImmutableMap.of(key, km); Map<String, Map<StaticBuffer, KCVMutation>> mutations = ImmutableMap.of( storeName1, keyColumnAndValue, storeName2, keyColumnAndValue); manager.mutateMany(mutations, directTx); directTx.commit(); KeySliceQuery query = new KeySliceQuery(key, col, nextCol); List<Entry> expected = ImmutableList.of(StaticArrayEntry.of(col, val)); Assert.assertEquals(expected, store1.getSlice(query, tx)); Assert.assertEquals(expected, store2.getSlice(query, tx)); }
@Test public void testMutateWithLockUsesConsistentTx() throws BackendException { final ImmutableList<Entry> adds = ImmutableList.of(StaticArrayEntry.of(DATA_COL, DATA_VAL)); final ImmutableList<StaticBuffer> deletions = ImmutableList.of(); final KeyColumn kc = new KeyColumn(LOCK_KEY, LOCK_COL); // 1. Acquire a lock backingLocker.writeLock(kc, consistentTx); // 2. Run a mutation // N.B. mutation coordinates do not overlap with the lock, but consistentTx should be used anyway // 2.1. Check locks & expected values before mutating data backingLocker.checkLocks(consistentTx); StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn()); KeySliceQuery expectedValueQuery = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf); expect(backingStore.getSlice(expectedValueQuery, consistentTx)) // expected value read must use strong consistency .andReturn(StaticArrayEntryList.of(StaticArrayEntry.of(LOCK_COL, LOCK_VAL))); // 2.2. Mutate data backingStore.mutate(DATA_KEY, adds, deletions, consistentTx); // writes by txs with locks must use strong consistency ctrl.replay(); // 1. Lock acquisition expectStore.acquireLock(LOCK_KEY, LOCK_COL, LOCK_VAL, expectTx); // 2. Mutate expectStore.mutate(DATA_KEY, adds, deletions, expectTx); }
@Test public void getSliceRespectsAllBoundsInclusionArguments() throws Exception { // Test case where endColumn=startColumn+1 StaticBuffer key = KeyColumnValueStoreUtil.longToByteBuffer(0); StaticBuffer columnBeforeStart = KeyColumnValueStoreUtil.longToByteBuffer(776); StaticBuffer columnStart = KeyColumnValueStoreUtil.longToByteBuffer(777); StaticBuffer columnEnd = KeyColumnValueStoreUtil.longToByteBuffer(778); StaticBuffer columnAfterEnd = KeyColumnValueStoreUtil.longToByteBuffer(779); // First insert four test Entries List<Entry> entries = Arrays.asList( StaticArrayEntry.of(columnBeforeStart, columnBeforeStart), StaticArrayEntry.of(columnStart, columnStart), StaticArrayEntry.of(columnEnd, columnEnd), StaticArrayEntry.of(columnAfterEnd, columnAfterEnd)); store.mutate(key, entries, KeyColumnValueStore.NO_DELETIONS, tx); tx.commit(); // getSlice() with only start inclusive tx = startTx(); List<Entry> result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd), tx); Assert.assertEquals(1, result.size()); Assert.assertEquals(777, KeyColumnValueStoreUtil.bufferToLong(result.get(0).getColumn())); }