private Optional<Long> readConservative(int shardNumber) { return sweepableTimestamps.nextSweepableTimestampPartition( conservative(shardNumber), progress.getLastSweptTimestamp(ShardAndStrategy.conservative(shardNumber)), Sweeper.CONSERVATIVE.getSweepTimestamp(timestampsSupplier)); }
private void assertSweepableCellsHasNoEntriesInPartitionOfTimestamp(long timestamp) { SweepBatch batch = sweepableCells.getBatchForPartition( ShardAndStrategy.conservative(CONS_SHARD), tsPartitionFine(timestamp), -1L, Long.MAX_VALUE); assertThat(batch.writes()).isEmpty(); }
@Test public void sweepCellOnlyOnceWhenInLastPartitionBeforeSweepTs() { immutableTs = 2 * TS_COARSE_GRANULARITY - TS_FINE_GRANULARITY; verify(spiedKvs, never()).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); enqueueWriteCommitted(TABLE_CONS, immutableTs - 1); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); verify(spiedKvs, times(1)).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); verify(spiedKvs, times(1)).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); }
private void assertNoEntriesInSweepableTimestampsBeforeSweepTimestamp() { assertThat(sweepableTimestamps .nextSweepableTimestampPartition(ShardAndStrategy.conservative(CONS_SHARD), -1L, getSweepTsCons())) .isEmpty(); }
@Test public void callingEnqueueAndSweepOnUninitializedSweeperThrows() { TargetedSweeper uninitializedSweeper = TargetedSweeper.createUninitializedForTest(null); assertThatThrownBy(() -> uninitializedSweeper.enqueue(ImmutableList.of())) .isInstanceOf(NotInitializedException.class) .hasMessageContaining("Targeted Sweeper"); assertThatThrownBy(() -> uninitializedSweeper.sweepNextBatch(ShardAndStrategy.conservative(0))) .isInstanceOf(NotInitializedException.class) .hasMessageContaining("Targeted Sweeper"); }
@Test public void sweepDisabledIsReportedInOutcome() { sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasTargetedOutcomeEqualTo(SweepOutcome.NOTHING_TO_SWEEP, 1L); enabled = false; sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasTargetedOutcomeEqualTo(SweepOutcome.DISABLED, 1L); }
@Test public void canReadNextTimestampIfProgressedBeyondButInSamePartitionForConservative() { progress.updateLastSweptTimestamp(conservative(shardCons), maxTsForFinePartition(TS_FINE_PARTITION) - 1); assertThat(readConservative(shardCons)).contains(TS_FINE_PARTITION); }
@Test public void noNextTimestampIfProgressedToEndOfPartitionForConservative() { progress.updateLastSweptTimestamp(conservative(shardCons), maxTsForFinePartition(TS_FINE_PARTITION)); assertThat(readConservative(shardCons)).isEmpty(); }
@Test public void sweepProgressesToJustBeforeSweepTsWhenNothingToSweep() { sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(getSweepTsCons() - 1L); }
@Test public void canReadNextWhenOtherShardsAndStrategiesProgressToEndOfPartitionForThorough() { progress.updateLastSweptTimestamp(thorough(shardCons), maxTsForFinePartition(TS2_FINE_PARTITION)); progress.updateLastSweptTimestamp(conservative(shardThor), maxTsForFinePartition(TS2_FINE_PARTITION)); progress.updateLastSweptTimestamp(conservative(shardCons), maxTsForFinePartition(TS2_FINE_PARTITION)); assertThat(readThorough(shardThor)).contains(TS2_FINE_PARTITION); }
@Test public void sweepProgressesToEndOfPartitionWhenFewValuesAndSweepTsLarge() { long writeTs = getSweepTsCons() - 3 * TS_FINE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, writeTs); enqueueWriteCommitted(TABLE_CONS, writeTs + 5); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(maxTsForFinePartition(tsPartitionFine(writeTs))); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertProgressUpdatedToTimestamp(getSweepTsCons() - 1L); }
@Test public void doesNotLeaveSentinelsIfTableDestroyed() { enqueueWriteCommitted(TABLE_CONS, 10); immutableTs = 11; unreadableTs = 11; spiedKvs.truncateTable(TABLE_CONS); assertThat(spiedKvs.getRange(TABLE_CONS, RangeRequest.all(), Long.MAX_VALUE)).isEmpty(); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(spiedKvs.getRange(TABLE_CONS, RangeRequest.all(), Long.MAX_VALUE)).isEmpty(); }
@Test public void sweepWithNoCandidatesBeforeSweepTimestampReportsNothingToSweep() { enqueueWriteCommitted(TABLE_CONS, getSweepTsCons()); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasTargetedOutcomeEqualTo(SweepOutcome.NOTHING_TO_SWEEP, 1L); }
@Test public void conservativeSweepAddsSentinelAndLeavesSingleValue() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); assertReadAtTimestampReturnsNothing(TABLE_CONS, LOW_TS); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); }
@Test public void sweepDeletesValuesWrittenOnPreCommitConditionFailure() { useOneSweepQueueShard(); long startTs = putWriteAndFailOnPreCommitConditionReturningStartTimestamp(SINGLE_WRITE); serializableTxManager.setUnreadableTimestamp(startTs + 1); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(0)); assertNoEntryForCellInKvs(TABLE_CONS, TEST_CELL); }
@Test public void getCorrectNextTimestampWhenMultipleCandidates() { for (long timestamp = 1000L; tsPartitionFine(timestamp) < 10L; timestamp += TS_FINE_GRANULARITY / 5) { writeToDefaultCellCommitted(sweepableTimestamps, timestamp, TABLE_CONS); } assertThat(readConservative(shardCons)).contains(tsPartitionFine(1000L)); progress.updateLastSweptTimestamp(conservative(shardCons), 2L * TS_FINE_GRANULARITY); assertThat(readConservative(shardCons)).contains(tsPartitionFine(2L * TS_FINE_GRANULARITY + 1000L)); setSweepTimestampAndGet(4 * TS_FINE_GRANULARITY); assertThat(readConservative(shardCons)).contains(tsPartitionFine(2L * TS_FINE_GRANULARITY + 1000L)); }
@Test public void sweepThrowsAwayWritesForDroppedTables() { useOneSweepQueueShard(); writeInTransactionAndGetStartTimestamp(SINGLE_WRITE); Long startTimestamp = writeInTransactionAndGetStartTimestamp(SINGLE_WRITE); keyValueService.dropTable(TABLE_CONS); serializableTxManager.setUnreadableTimestamp(startTimestamp + 1); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(0)); }
@Test public void canReadNextWhenOtherShardsAndStrategiesProgressToEndOfPartitionForConservative() { progress.updateLastSweptTimestamp(thorough(shardCons), maxTsForFinePartition(TS_FINE_PARTITION)); progress.updateLastSweptTimestamp(conservative(shardThor), maxTsForFinePartition(TS_FINE_PARTITION)); progress.updateLastSweptTimestamp(thorough(shardThor), maxTsForFinePartition(TS_FINE_PARTITION)); assertThat(readConservative(shardCons)).contains(TS_FINE_PARTITION); }
@Test public void conservativeSweepDeletesLowerValue() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); }
@Test public void onlySweepsOneBatchAtATime() { enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS2); enqueueWriteCommitted(TABLE_CONS, TS_FINE_GRANULARITY); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertReadAtTimestampReturnsSentinel(TABLE_CONS, LOW_TS + 1); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, LOW_TS2); assertTestValueEnqueuedAtGivenTimestampStillPresent(TABLE_CONS, TS_FINE_GRANULARITY); }