@Test public void canSweepAtMinimumTime() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); enqueueWriteCommitted(TABLE_CONS, LOW_TS3); runConservativeSweepAtTimestamp(Long.MIN_VALUE); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); assertProgressUpdatedToTimestamp(SweepQueueUtils.INITIAL_TIMESTAMP); }
@Test public void canSweepAtMaximumTime() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); enqueueWriteCommitted(TABLE_CONS, LOW_TS3); runConservativeSweepAtTimestamp(Long.MAX_VALUE); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS3); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS3); }
@Test public void thoroughSweepCallsFollower() { enqueueWriteCommitted(TABLE_THOR, LOW_TS); enqueueWriteCommitted(TABLE_THOR, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(THOR_SHARD)); ArgumentCaptor<Set> captor = ArgumentCaptor.forClass(Set.class); verify(mockFollower, times(1)).run(eq(TABLE_THOR), captor.capture()); assertThat(Iterables.getOnlyElement(captor.getAllValues())).containsExactly(DEFAULT_CELL); }
@Test public void conservativeSweepCallsFollower() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); ArgumentCaptor<Set> captor = ArgumentCaptor.forClass(Set.class); verify(mockFollower, times(1)).run(eq(TABLE_CONS), captor.capture()); assertThat(Iterables.getOnlyElement(captor.getAllValues())).containsExactly(DEFAULT_CELL); }
@Test public void enqueueUpdatesNumberOfShards() { assertThat(AtlasDbConstants.DEFAULT_SWEEP_QUEUE_SHARDS).isLessThan(DEFAULT_SHARDS); assertThat(progress.getNumberOfShards()).isEqualTo(AtlasDbConstants.DEFAULT_SWEEP_QUEUE_SHARDS); enqueueWriteCommitted(TABLE_CONS, LOW_TS); assertThat(progress.getNumberOfShards()).isEqualTo(DEFAULT_SHARDS); }
@Test public void onlySweepsOneBatchAtATime() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); enqueueWriteCommitted(TABLE_CONS, TS_FINE_GRANULARITY); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, TS_FINE_GRANULARITY); }
@Test public void sweepStrategyNothingDoesNotPersistAnything() { enqueueWriteCommitted(TABLE_NOTH, LOW_TS); enqueueWriteCommitted(TABLE_NOTH, LOW_TS2); verify(spiedKvs, times(2)).put(eq(TABLE_NOTH), anyMap(), anyLong()); verify(spiedKvs, times(2)).put(any(TableReference.class), anyMap(), anyLong()); }
@Test public void sweepCellOnlyOnceWhenInLastPartitionBeforeSweepTs() { immutableTs = 2 * TS_COARSE_GRANULARITY - TS_FINE_GRANULARITY; verify(spiedKvs, never()).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); enqueueWriteCommitted(TABLE_CONS, immutableTs - 1); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); verify(spiedKvs, times(1)).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); verify(spiedKvs, times(1)).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); }
@Test public void sweepStrategyNothingDoesNotUpdateMetrics() { enqueueWriteCommitted(TABLE_NOTH, LOW_TS); enqueueWriteCommitted(TABLE_NOTH, LOW_TS2); assertThat(metricsManager).hasEnqueuedWritesConservativeEqualTo(0); assertThat(metricsManager).hasEnqueuedWritesThoroughEqualTo(0); }
@Test public void conservativeSweepDeletesAllButLatestWithSingleDeleteAllTimestamps() { long lastWriteTs = TS_FINE_GRANULARITY - 1; for (long i = 1; i <= lastWriteTs; i++) { enqueueWriteCommitted(TABLE_CONS, i); } sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, lastWriteTs); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, lastWriteTs); verify(spiedKvs, times(1)).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); }
@Test public void thoroughSweepDeletesAllButLatestWithSingleDeleteAllTimestampsIncludingSentinels() { long lastWriteTs = TS_FINE_GRANULARITY - 1; for (long i = 1; i <= lastWriteTs; i++) { enqueueWriteCommitted(TABLE_THOR, i); } sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(THOR_SHARD)); assertReadAtTimestampReturnsNothing(TABLE_THOR, lastWriteTs); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, lastWriteTs); verify(spiedKvs, times(1)).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(true)); }
@Test public void thoroughSweepDeletesLowerValue() { enqueueWriteCommitted(TABLE_THOR, LOW_TS); enqueueWriteCommitted(TABLE_THOR, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(THOR_SHARD)); assertReadAtTimestampReturnsNothing(TABLE_THOR, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_THOR, LOW_TS2); }
@Test public void conservativeSweepDeletesLowerValue() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); }
@Test public void thoroughSweepDeletesExistingSentinel() { spiedKvs.addGarbageCollectionSentinelValues(TABLE_THOR, ImmutableList.of(DEFAULT_CELL)); assertReadAtTimestampReturnsSentinel(TABLE_THOR, 0L); enqueueWriteCommitted(TABLE_THOR, 10L); sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(THOR_SHARD)); assertReadAtTimestampReturnsNothing(TABLE_THOR, 0L); }
@Test public void sweepProgressesToEndOfPartitionWhenFewValuesAndSweepTsLarge() { long writeTs = getSweepTsCons() - 3 * TS_FINE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, writeTs); enqueueWriteCommitted(TABLE_CONS, writeTs + 5); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(maxTsForFinePartition(tsPartitionFine(writeTs))); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(getSweepTsCons() - 1L); }
@Test public void doesNotLeaveSentinelsIfTableDestroyed() { enqueueWriteCommitted(TABLE_CONS, 10); immutableTs = 11; unreadableTs = 11; spiedKvs.truncateTable(TABLE_CONS); assertThat(spiedKvs.getRange(TABLE_CONS, RangeRequest.all(), Long.MAX_VALUE)).isEmpty(); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(spiedKvs.getRange(TABLE_CONS, RangeRequest.all(), Long.MAX_VALUE)).isEmpty(); }
@Test public void conservativeSweepAddsSentinelAndLeavesSingleValue() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); assertReadAtTimestampReturnsNothing(TABLE_CONS, LOW_TS); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); }
@Test public void sweepWithNoCandidatesBeforeSweepTimestampReportsNothingToSweep() { enqueueWriteCommitted(TABLE_CONS, getSweepTsCons()); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasTargetedOutcomeEqualTo(SweepOutcome.NOTHING_TO_SWEEP, 1L); }
@Test public void sweepDeletesTombstonesWhenWriteHasHigherTimestamp() { enqueueTombstone(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); assertReadAtTimestampReturnsTombstoneAtTimestamp(TABLE_CONS, LOW_TS + 1, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); }
@Test public void sweepDeletesWritesWhenTombstoneHasHigherTimestamp() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueTombstone(TABLE_CONS, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); assertReadAtTimestampReturnsTombstoneAtTimestamp(TABLE_CONS, LOW_TS2 + 1, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertReadAtTimestampReturnsTombstoneAtTimestamp(TABLE_CONS, LOW_TS2 + 1, LOW_TS2); }