private long getLastSweptTs(TimestampsToSweep startTsCommitted, Iterator<Map.Entry<Cell, Value>> resultIterator, long partitionFine, long maxTsExclusive) { if (startTsCommitted.processedAll() && exhaustedAllColumns(resultIterator)) { return lastGuaranteedSwept(partitionFine, maxTsExclusive); } else { return startTsCommitted.maxSwept(); } }
@Override public void enqueue(List<WriteInfo> writes) { assertInitialized(); queue.enqueue(writes); }
@Test public void conservativeSweepDeletesLowerValue() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); }
@Test public void sweepProgressesToEndOfPartitionWhenFewValuesAndSweepTsLarge() { long writeTs = getSweepTsCons() - 3 * TS_FINE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, writeTs); enqueueWriteCommitted(TABLE_CONS, writeTs + 5); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(maxTsForFinePartition(tsPartitionFine(writeTs))); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(getSweepTsCons() - 1L); }
@Test public void sweepDeletesTombstonesWhenWriteHasHigherTimestamp() { enqueueTombstone(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); assertReadAtTimestampReturnsTombstoneAtTimestamp(TABLE_CONS, LOW_TS + 1, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); }
@Test public void thoroughSweepDeletesLowerValue() { enqueueWriteCommitted(TABLE_THOR, LOW_TS); enqueueWriteCommitted(TABLE_THOR, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(THOR_SHARD)); assertReadAtTimestampReturnsNothing(TABLE_THOR, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS2); }
private void assertSweepableCellsHasEntryForTimestamp(long timestamp) { SweepBatch batch = sweepableCells.getBatchForPartition( ShardAndStrategy.conservative(CONS_SHARD), tsPartitionFine(timestamp), -1L, timestamp + 1); assertThat(batch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, timestamp)); }
@Test public void sweepProgressesToJustBeforeSweepTsWhenNothingToSweep() { sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(getSweepTsCons() - 1L); }
private void runConservativeSweepAtTimestamp(long desiredSweepTimestamp) { runWithConservativeSweepTimestamp(() -> sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)), desiredSweepTimestamp); }
private Optional<Long> readConservative(int shardNumber) { return sweepableTimestamps.nextSweepableTimestampPartition( conservative(shardNumber), progress.getLastSweptTimestamp(ShardAndStrategy.conservative(shardNumber)), Sweeper.CONSERVATIVE.getSweepTimestamp(timestampsSupplier)); }
static SweepBatch of(Collection<WriteInfo> writes, DedicatedRows dedicatedRows, long timestamp) { return ImmutableSweepBatch.builder() .writes(writes) .dedicatedRows(dedicatedRows) .lastSweptTimestamp(timestamp).build(); } }
private WriteBatch getWrites(SweepableCellsRow row, SweepableCellsTable.SweepableCellsColumn col, Value value) { if (isReferenceToDedicatedRows(col)) { return writesFromDedicated(row, col); } else { return WriteBatch.single(getWriteInfo(getTimestamp(row, col), value)); } }
static TimestampsToSweep of(SortedSet<Long> list, long max, boolean processedAll) { return ImmutableTimestampsToSweep.builder() .timestampsDescending(list) .maxSwept(max) .processedAll(processedAll) .build(); } }
private PartitionInfo getPartitionInfo(WriteInfo write, int shards) { return PartitionInfo.of(write.toShard(shards), isConservative(write), write.timestamp()); }
@Before @Override public void setup() { super.setup(); progress = new ShardProgress(spiedKvs); sweepableTimestamps = new SweepableTimestamps(spiedKvs, partitioner); shardCons = writeToDefaultCellCommitted(sweepableTimestamps, TS, TABLE_CONS); shardThor = writeToDefaultCellCommitted(sweepableTimestamps, TS2, TABLE_THOR); }
public static SweepQueue create( TargetedSweepMetrics metrics, KeyValueService kvs, TimelockService timelock, Supplier<Integer> shardsConfig, TransactionService transaction, TargetedSweepFollower follower) { return new SweepQueue(SweepQueueFactory.create(metrics, kvs, timelock, shardsConfig, transaction), follower); }
@Test public void onlySweepsOneBatchAtATime() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); enqueueWriteCommitted(TABLE_CONS, TS_FINE_GRANULARITY); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, TS_FINE_GRANULARITY); }
@Test public void conservativeSweepAddsSentinelAndLeavesSingleValue() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); assertReadAtTimestampReturnsNothing(TABLE_CONS, LOW_TS); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); }