@Test public void canSweepAtMinimumTime() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); enqueueWriteCommitted(TABLE_CONS, LOW_TS3); runConservativeSweepAtTimestamp(Long.MIN_VALUE); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); assertProgressUpdatedToTimestamp(SweepQueueUtils.INITIAL_TIMESTAMP); }
@Test public void remembersProgressWhenSweepTimestampAdvances() { long baseSweepTs = getSweepTsCons(); long oldPartitionTs = baseSweepTs - 5; long newPartitionFirstTs = baseSweepTs + 5; long newPartitionSecondTs = baseSweepTs + 10; enqueueWriteCommitted(TABLE_CONS, oldPartitionTs); enqueueWriteCommitted(TABLE_CONS, newPartitionFirstTs); enqueueWriteCommitted(TABLE_CONS, newPartitionSecondTs); assertReadAtTimestampReturnsNothing(TABLE_CONS, oldPartitionTs); runConservativeSweepAtTimestamp(baseSweepTs + 7); runConservativeSweepAtTimestamp(baseSweepTs + 7); assertReadAtTimestampReturnsSentinel(TABLE_CONS, oldPartitionTs + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, newPartitionFirstTs); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, newPartitionSecondTs); runConservativeSweepAtTimestamp(newPartitionSecondTs + 1); assertReadAtTimestampReturnsSentinel(TABLE_CONS, newPartitionSecondTs); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, newPartitionSecondTs); }
@Test public void sweepDeletesTombstonesWhenWriteHasHigherTimestamp() { enqueueTombstone(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); assertReadAtTimestampReturnsTombstoneAtTimestamp(TABLE_CONS, LOW_TS + 1, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); }
@Test public void canSweepAtMaximumTime() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); enqueueWriteCommitted(TABLE_CONS, LOW_TS3); runConservativeSweepAtTimestamp(Long.MAX_VALUE); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS3); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS3); }
private void writeValuesAroundSweepTimestampAndSweepAndCheck(long sweepTimestamp, int sweepIterations) { enqueueWriteCommitted(TABLE_CONS, sweepTimestamp - 10); enqueueWriteCommitted(TABLE_CONS, sweepTimestamp - 5); enqueueWriteCommitted(TABLE_CONS, sweepTimestamp + 5); IntStream.range(0, sweepIterations) .forEach(unused -> sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD))); assertReadAtTimestampReturnsSentinel(TABLE_CONS, sweepTimestamp - 5); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, sweepTimestamp - 5); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, sweepTimestamp + 5); }
@Test public void sweepableCellsGetsScrubbedWheneverLastSweptInNewPartition() { long tsSecondPartitionFine = LOW_TS + TS_FINE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS + 1L); enqueueAtLeastThresholdWritesInDefaultShardWithStartTs(100, LOW_TS + 2L); putTimestampIntoTransactionTable(LOW_TS + 2L, LOW_TS + 2L); enqueueWriteCommitted(TABLE_CONS, tsSecondPartitionFine); enqueueWriteCommitted(TABLE_CONS, getSweepTsCons()); // last swept timestamp: TS_FINE_GRANULARITY - 1 sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertSweepableCellsHasEntryForTimestamp(LOW_TS + 1); assertSweepableCellsHasEntryForTimestamp(tsSecondPartitionFine); assertSweepableCellsHasEntryForTimestamp(getSweepTsCons()); // last swept timestamp: 2 * TS_FINE_GRANULARITY - 1 sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertSweepableCellsHasNoEntriesInPartitionOfTimestamp(LOW_TS + 1); assertSweepableCellsHasEntryForTimestamp(tsSecondPartitionFine); assertSweepableCellsHasEntryForTimestamp(getSweepTsCons()); // last swept timestamp: largestBeforeSweepTs sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertSweepableCellsHasNoEntriesInPartitionOfTimestamp(LOW_TS + 1); assertSweepableCellsHasNoEntriesInPartitionOfTimestamp(tsSecondPartitionFine); assertSweepableCellsHasEntryForTimestamp(getSweepTsCons()); assertSweepableCellsHasNoDedicatedRowsForShard(CONS_SHARD); }
long tsFineTwo = LOW_TS + TS_FINE_GRANULARITY; long tsFineFour = LOW_TS + 3 * TS_FINE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, LOW_TS); punchTimeAtTimestamp(100L, LOW_TS); enqueueWriteCommitted(TABLE_CONS, tsFineTwo); punchTimeAtTimestamp(200L, tsFineTwo); enqueueWriteCommitted(TABLE_CONS, tsFineFour); punchTimeAtTimestamp(300L, tsFineFour); enqueueWriteCommitted(TABLE_CONS, tsFineFour + 1L); punchTimeAtTimestamp(400L, tsFineFour + 1L); enqueueWriteCommitted(TABLE_CONS, 5 * TS_FINE_GRANULARITY); punchTimeAtTimestamp(3000L, 5 * TS_FINE_GRANULARITY); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); assertThat(metricsManager).hasTombstonesPutConservativeEqualTo(1); assertReadAtTimestampReturnsSentinel(TABLE_CONS, tsFineTwo); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, tsFineTwo); assertThat(metricsManager).hasTombstonesPutConservativeEqualTo(2); assertReadAtTimestampReturnsSentinel(TABLE_CONS, tsFineFour + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, tsFineFour + 1); assertThat(metricsManager).hasTombstonesPutConservativeEqualTo(3); assertThat(metricsManager).hasEntriesReadConservativeEqualTo(4); assertThat(metricsManager).hasLastSweptTimestampConservativeEqualTo(maxTsForFinePartition(3)); setTimelockTime(5000L);
@Test public void thoroughSweepDeletesLowerValue() { enqueueWriteCommitted(TABLE_THOR, LOW_TS); enqueueWriteCommitted(TABLE_THOR, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(THOR_SHARD)); assertReadAtTimestampReturnsNothing(TABLE_THOR, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS2); }
@Test public void conservativeSweepAddsSentinelAndLeavesSingleValue() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); assertReadAtTimestampReturnsNothing(TABLE_CONS, LOW_TS); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); }
@Test public void sweepProgressesAcrossCoarsePartitions() { long tsCoarseTwo = LOW_TS + TS_FINE_GRANULARITY + TS_COARSE_GRANULARITY; long tsCoarseFour = LOW_TS + 3 * TS_COARSE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, tsCoarseTwo); enqueueWriteCommitted(TABLE_CONS, tsCoarseFour); enqueueWriteCommitted(TABLE_CONS, tsCoarseFour + 1L); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, tsCoarseTwo); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, tsCoarseTwo); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, tsCoarseFour + 1); assertReadAtTimestampReturnsValue(TABLE_CONS, tsCoarseFour + 2, tsCoarseFour + 1); }
@Test public void thoroughSweepDeletesTombstoneIfLatestWrite() { enqueueTombstone(TABLE_THOR, LOW_TS); enqueueTombstone(TABLE_THOR, LOW_TS2); assertReadAtTimestampReturnsTombstoneAtTimestamp(TABLE_THOR, LOW_TS + 1, LOW_TS); assertReadAtTimestampReturnsTombstoneAtTimestamp(TABLE_THOR, LOW_TS2 + 1, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(THOR_SHARD)); assertReadAtTimestampReturnsNothing(TABLE_THOR, LOW_TS + 1); assertReadAtTimestampReturnsNothing(TABLE_THOR, LOW_TS2 + 1); }
@Test public void sweepProgressesToEndOfPartitionWhenFewValuesAndSweepTsLarge() { long writeTs = getSweepTsCons() - 3 * TS_FINE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, writeTs); enqueueWriteCommitted(TABLE_CONS, writeTs + 5); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(maxTsForFinePartition(tsPartitionFine(writeTs))); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(getSweepTsCons() - 1L); }
@Test public void thoroughSweepDeletesExistingSentinel() { spiedKvs.addGarbageCollectionSentinelValues(TABLE_THOR, ImmutableList.of(DEFAULT_CELL)); assertReadAtTimestampReturnsSentinel(TABLE_THOR, 0L); enqueueWriteCommitted(TABLE_THOR, 10L); sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(THOR_SHARD)); assertReadAtTimestampReturnsNothing(TABLE_THOR, 0L); }
@Test public void sweepableTimestampsGetsScrubbedWhenNoMoreToSweepButSweepTsInNewCoarsePartition() { long tsSecondPartitionFine = LOW_TS + TS_FINE_GRANULARITY; long largestFirstPartitionCoarse = TS_COARSE_GRANULARITY - 1L; enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, tsSecondPartitionFine); enqueueWriteCommitted(TABLE_CONS, largestFirstPartitionCoarse); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertLowestFinePartitionInSweepableTimestampsEquals(tsPartitionFine(LOW_TS)); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertLowestFinePartitionInSweepableTimestampsEquals(tsPartitionFine(LOW_TS)); // after this sweep we progress to sweepTsConservative - 1 sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertNoEntriesInSweepableTimestampsBeforeSweepTimestamp(); }
@Test public void multipleSweepersSweepDifferentShardsAndCallUnlockAfterwards() throws InterruptedException { int shards = 128; int sweepers = 8; int threads = shards / sweepers; TimelockService stickyLockService = createStickyLockService(); createAndInitializeSweepersAndWaitForOneBackgroundIteration(sweepers, shards, threads, stickyLockService); for (int i = 0; i < shards; i++) { assertProgressUpdatedToTimestamp(maxTsForFinePartition(tsPartitionFine(unreadableTs - 1)), i); verify(stickyLockService, times(1)).unlock(ImmutableSet.of(LockToken.of(new UUID(i, 0L)))); } // minimum: all threads on one host succeed, then on another, etc: // threads + threads * 2 + ... + threads * swepers verify(stickyLockService, atLeast(threads * sweepers * (sweepers - 1) / 2)) .lock(any(LockRequest.class)); // maximum: all but one succeed on each host, and only then those succeed: // shards + shards - 1 + ... + shards - (sweepers - 1) verify(stickyLockService, atMost(sweepers * shards - sweepers * (sweepers - 1) / 2)) .lock(any(LockRequest.class)); }
@Test public void doNotDeleteAnythingAfterEntryWithCommitTsAfterSweepTs() { immutableTs = 1000L; enqueueWriteUncommitted(TABLE_CONS, 900); enqueueWriteUncommitted(TABLE_CONS, 920); enqueueWriteCommitedAt(TABLE_CONS, 950, 2000); enqueueWriteUncommitted(TABLE_CONS, 970); enqueueWriteUncommitted(TABLE_CONS, 1110); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(920L); verify(spiedKvs, never()).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); ArgumentCaptor<Multimap> multimap = ArgumentCaptor.forClass(Multimap.class); verify(spiedKvs, times(1)).delete(eq(TABLE_CONS), multimap.capture()); assertThat(multimap.getValue().keySet()).containsExactly(DEFAULT_CELL); assertThat(multimap.getValue().values()).containsExactly(900L, 920L); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(920L); verify(spiedKvs, never()).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); verify(spiedKvs, times(1)).delete(any(TableReference.class), any(Multimap.class)); assertReadAtTimestampReturnsValue(TABLE_CONS, 1500L, 1110L); immutableTs = 2009L; sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(2009L - 1L); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, times(1)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getValue()).containsValue(950L); assertReadAtTimestampReturnsValue(TABLE_CONS, 1500L, 950L); }
@Test public void canSweepAtMinimumTimeWithNoWrites() { runConservativeSweepAtTimestamp(Long.MIN_VALUE); assertProgressUpdatedToTimestamp(SweepQueueUtils.INITIAL_TIMESTAMP); }
@Test public void sweepProgressesToJustBeforeSweepTsWhenNothingToSweep() { sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(getSweepTsCons() - 1L); }
@Test public void sweepableTimestampsGetsScrubbedWhenLastSweptProgressesInNewCoarsePartition() { long tsSecondPartitionFine = LOW_TS + TS_FINE_GRANULARITY; long largestFirstPartitionCoarse = TS_COARSE_GRANULARITY - 1L; long thirdPartitionCoarse = 2 * TS_COARSE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, tsSecondPartitionFine); enqueueWriteCommitted(TABLE_CONS, largestFirstPartitionCoarse); enqueueWriteCommitted(TABLE_CONS, thirdPartitionCoarse); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertLowestFinePartitionInSweepableTimestampsEquals(tsPartitionFine(LOW_TS)); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertLowestFinePartitionInSweepableTimestampsEquals(tsPartitionFine(LOW_TS)); // after this sweep we progress to thirdPartitionCoarse - 1 sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertLowestFinePartitionInSweepableTimestampsEquals(tsPartitionFine(thirdPartitionCoarse)); }
private void assertTestValueEnqueuedAtGivenTimestampStillPresent(TableReference tableRef, long timestamp) { assertReadAtTimestampReturnsValue(tableRef, timestamp + 1, timestamp); }