public static <E> EntryList ofBytes(Iterable<E> elements, StaticArrayEntry.GetColVal<E,byte[]> getter) { return of(elements, getter, StaticArrayEntry.ByteArrayHandler.INSTANCE); }
@Override public Entry get(int index) { Preconditions.checkPositionIndex(index,limitAndValuePos.length); int offset = index>0?getLimit(limitAndValuePos[index-1]):0; Map<EntryMetaData,Object> metadata = EntryMetaData.EMPTY_METADATA; if (hasMetaData()) { metadata = new EntryMetaData.Map(); offset = parseMetaData(metadata,offset); } return new StaticEntry(index, offset, getLimit(limitAndValuePos[index]), getValuePos(limitAndValuePos[index]),metadata); }
private int parseMetaData(Map<EntryMetaData,Object> metadata, int baseOffset) { assert hasMetaData(); for (EntryMetaData meta : metaDataSchema) { MetaDataSerializer s = getSerializer(meta); Object d = s.read(data,baseOffset); baseOffset+=s.getByteLength(d); metadata.put(meta,d); } return baseOffset; }
assert colSize>0; int valueSize = dataHandler.getSize(val); int metaDataSize = getMetaDataSize(metadataSchema,element,getter); data = ensureSpace(data,offset,colSize+valueSize+metaDataSize); offset=writeMetaData(data,offset,metadataSchema,element,getter); offset+=valueSize; limitAndValuePos = ensureSpace(limitAndValuePos,pos); limitAndValuePos[pos]= getOffsetAndValue(offset,colSize); //valuePosition = colSize pos++; return new StaticArrayEntryList(data,limitAndValuePos,metadataSchema);
public static EntryList convert(RecordIterator<KeyValueEntry> iterator) throws BackendException { try { return StaticArrayEntryList.ofStaticBuffer(iterator, KVEntryGetter.INSTANCE); } finally { try { iterator.close(); } catch (IOException e) { throw new TemporaryBackendException(e); } } }
public static EntryList of(Iterable<Entry> entries) { Preconditions.checkNotNull(entries); int num=0; int dataLength=0; EntryMetaData[] metadataSchema = null; for (Entry entry : entries) { num++; dataLength+=entry.length(); if (metadataSchema==null) metadataSchema=StaticArrayEntry.ENTRY_GETTER.getMetaSchema(entry); dataLength+=getMetaDataSize(metadataSchema,entry,StaticArrayEntry.ENTRY_GETTER); } if (num==0) return EMPTY_LIST; byte[] data = new byte[dataLength]; long[] limitAndValuePos = new long[num]; int pos=0; CopyFactory cpf = new CopyFactory(data); for (Entry entry : entries) { cpf.addMetaData(metadataSchema,entry); entry.as(cpf); limitAndValuePos[pos]= getOffsetAndValue(cpf.dataOffset,entry.getValuePosition()); pos++; } assert cpf.dataOffset==data.length; assert pos==limitAndValuePos.length; return new StaticArrayEntryList(data,limitAndValuePos,metadataSchema); }
private static<D,K> int getMetaDataSize(EntryMetaData[] schema, D entry, StaticArrayEntry.GetColVal<D,K> metaGetter) { int dataSize = 0; if (schema.length>0) { assert schema.length==metaGetter.getMetaSchema(entry).length; for (EntryMetaData meta : schema) { Object data = metaGetter.getMetaData(entry,meta); assert data!=null; dataSize+=getSerializer(meta).getByteLength(data); } } return dataSize; }
dataLength+=dataHandler.getSize(getter.getValue(element)); if (metadataSchema==null) metadataSchema=getter.getMetaSchema(element); dataLength+=getMetaDataSize(metadataSchema,element,getter); if (element==null) throw new IllegalArgumentException("Unexpected null element in result set"); offset=writeMetaData(data,offset,metadataSchema,element,getter); offset+=dataHandler.getSize(val); limitAndValuePos[pos]= getOffsetAndValue(offset,valuePos); pos++; return new StaticArrayEntryList(data,limitAndValuePos,metadataSchema);
private EntryList convert(RecordIterator<KeyValueEntry> entries) throws BackendException { try { return StaticArrayEntryList.ofStaticBuffer(entries,kvEntryGetter); } finally { try { entries.close(); } catch (IOException e) { /* * IOException could be permanent or temporary. Choosing temporary * allows useful retries of transient failures but also allows * futile retries of permanent failures. */ throw new TemporaryBackendException(e); } } }
private static<D,K> int writeMetaData(byte[] data, int startPos, EntryMetaData[] schema, D entry, StaticArrayEntry.GetColVal<D,K> metaGetter) { if (schema.length==0) return startPos; for (EntryMetaData meta : schema) { Object d = metaGetter.getMetaData(entry,meta); assert d!=null; MetaDataSerializer s = getSerializer(meta); s.write(data,startPos,d); startPos+=s.getByteLength(d); } return startPos; }
public static <E> EntryList ofByteBuffer(Iterator<E> elements, StaticArrayEntry.GetColVal<E,ByteBuffer> getter) { return of(elements, getter, StaticArrayEntry.ByteBufferHandler.INSTANCE); }
@Override public void run() { try { while (keyIterator.hasNext()) { StaticBuffer key = keyIterator.next(); RecordIterator<Entry> entries = keyIterator.getEntries(); if (!keyFilter.test(key)) continue; EntryList entryList = StaticArrayEntryList.ofStaticBuffer(entries, StaticArrayEntry.ENTRY_GETTER); queue.put(new SliceResult(query, key, entryList)); } finished = true; } catch (InterruptedException e) { log.error("Data-pulling thread interrupted while waiting on queue or data", e); } catch (Throwable e) { log.error("Could not load data from storage: {}",e); } finally { try { keyIterator.close(); } catch (IOException e) { log.warn("Could not close storage iterator ", e); } } }
public static <E> EntryList ofByteBuffer(Iterable<E> elements, StaticArrayEntry.GetColVal<E,ByteBuffer> getter) { return of(elements, getter, StaticArrayEntry.ByteBufferHandler.INSTANCE); }
private static EntryList fromResultSet(final ResultSet resultSet, final GetColVal<Tuple3<StaticBuffer, StaticBuffer, Row>, StaticBuffer> getter) { final Lazy<ArrayList<Row>> lazyList = Lazy.of(() -> Lists.newArrayList(resultSet)); // Use the Iterable overload of ofByteBuffer as it's able to allocate // the byte array up front. // To ensure that the Iterator instance is recreated, it is created // within the closure otherwise // the same iterator would be reused and would be exhausted. return StaticArrayEntryList.ofStaticBuffer(() -> Iterator.ofAll(lazyList.get()).map(row -> Tuple.of( StaticArrayBuffer.of(row.getBytes(COLUMN_COLUMN_NAME)), StaticArrayBuffer.of(row.getBytes(VALUE_COLUMN_NAME)), row)), getter); }
public static <E> EntryList ofStaticBuffer(Iterable<E> elements, StaticArrayEntry.GetColVal<E,StaticBuffer> getter) { return of(elements, getter, StaticArrayEntry.StaticBufferHandler.INSTANCE); }
public static <E> EntryList ofStaticBuffer(Iterator<E> elements, StaticArrayEntry.GetColVal<E,StaticBuffer> getter) { return of(elements, getter, StaticArrayEntry.StaticBufferHandler.INSTANCE); }
public static EntryList of(Entry... entries) { return of(Arrays.asList(entries)); }
public EntryList getSubset(final SliceQuery otherQuery, final EntryList otherResult) { assert otherQuery.subsumes(this); int pos = Collections.binarySearch(otherResult, sliceStart); if (pos < 0) pos = -pos - 1; final List<Entry> result = new ArrayList<>(); for (; pos < otherResult.size() && result.size() < getLimit(); pos++) { Entry e = otherResult.get(pos); if (e.getColumnAs(StaticBuffer.STATIC_FACTORY).compareTo(sliceEnd) < 0) result.add(e); else break; } return StaticArrayEntryList.of(result); }
@Test public void testMutateManyWithLockUsesConsistentTx() throws BackendException { final ImmutableList<Entry> adds = ImmutableList.of(StaticArrayEntry.of(DATA_COL, DATA_VAL)); final ImmutableList<StaticBuffer> deletions = ImmutableList.of(); Map<String, Map<StaticBuffer, KCVMutation>> mutations = ImmutableMap.of(STORE_NAME, ImmutableMap.of(DATA_KEY, new KCVMutation(adds, deletions))); final KeyColumn kc = new KeyColumn(LOCK_KEY, LOCK_COL); // Acquire a lock backingLocker.writeLock(kc, consistentTx); // 2. Run mutateMany // 2.1. Check locks & expected values before mutating data backingLocker.checkLocks(consistentTx); StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn()); KeySliceQuery expectedValueQuery = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf); expect(backingStore.getSlice(expectedValueQuery, consistentTx)) // expected value read must use strong consistency .andReturn(StaticArrayEntryList.of(StaticArrayEntry.of(LOCK_COL, LOCK_VAL))); // 2.2. Run mutateMany on backing manager to modify data backingManager.mutateMany(mutations, consistentTx); // writes by txs with locks must use strong consistency ctrl.replay(); // Lock acquisition expectStore.acquireLock(LOCK_KEY, LOCK_COL, LOCK_VAL, expectTx); // Mutate expectManager.mutateMany(mutations, expectTx); }
@Test public void testMutateWithLockUsesConsistentTx() throws BackendException { final ImmutableList<Entry> adds = ImmutableList.of(StaticArrayEntry.of(DATA_COL, DATA_VAL)); final ImmutableList<StaticBuffer> deletions = ImmutableList.of(); final KeyColumn kc = new KeyColumn(LOCK_KEY, LOCK_COL); // 1. Acquire a lock backingLocker.writeLock(kc, consistentTx); // 2. Run a mutation // N.B. mutation coordinates do not overlap with the lock, but consistentTx should be used anyway // 2.1. Check locks & expected values before mutating data backingLocker.checkLocks(consistentTx); StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn()); KeySliceQuery expectedValueQuery = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf); expect(backingStore.getSlice(expectedValueQuery, consistentTx)) // expected value read must use strong consistency .andReturn(StaticArrayEntryList.of(StaticArrayEntry.of(LOCK_COL, LOCK_VAL))); // 2.2. Mutate data backingStore.mutate(DATA_KEY, adds, deletions, consistentTx); // writes by txs with locks must use strong consistency ctrl.replay(); // 1. Lock acquisition expectStore.acquireLock(LOCK_KEY, LOCK_COL, LOCK_VAL, expectTx); // 2. Mutate expectStore.mutate(DATA_KEY, adds, deletions, expectTx); }