/** * Updates the persisted last swept timestamp for the given shard and strategy to timestamp if it is greater than * the currently persisted last swept timestamp. * * @param shardAndStrategy shard and strategy to update for * @param timestamp timestamp to update to * @return the latest known persisted sweep timestamp for the shard and strategy */ public long updateLastSweptTimestamp(ShardAndStrategy shardAndStrategy, long timestamp) { return increaseValueFromToAtLeast(shardAndStrategy, getLastSweptTimestamp(shardAndStrategy), timestamp); }
@Test public void updatingTimestampForOneShardDoesNotAffectOthers() { assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(INITIAL_TIMESTAMP); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TWENTY)).isEqualTo(INITIAL_TIMESTAMP); progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 1024L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TWENTY)).isEqualTo(INITIAL_TIMESTAMP); progress.updateLastSweptTimestamp(CONSERVATIVE_TWENTY, 512L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TWENTY)).isEqualTo(512L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(1024L); }
@Test public void updatingTimestampForOneConsistencyDoesNotAffectOther() { assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(INITIAL_TIMESTAMP); assertThat(progress.getLastSweptTimestamp(THOROUGH_TEN)).isEqualTo(INITIAL_TIMESTAMP); progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 128L); assertThat(progress.getLastSweptTimestamp(THOROUGH_TEN)).isEqualTo(INITIAL_TIMESTAMP); progress.updateLastSweptTimestamp(THOROUGH_TEN, 32L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(128L); assertThat(progress.getLastSweptTimestamp(THOROUGH_TEN)).isEqualTo(32L); }
@Test public void canReadInitialSweptTimestamp() { assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(INITIAL_TIMESTAMP); }
private void assertProgressUpdatedToTimestamp(long ts, int shard) { assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(shard))).isEqualTo(ts); }
@Test public void updatingTimestampsDoesNotAffectShardsAndViceVersa() { assertThat(progress.getNumberOfShards()).isEqualTo(AtlasDbConstants.DEFAULT_SWEEP_QUEUE_SHARDS); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(INITIAL_TIMESTAMP); assertThat(progress.getLastSweptTimestamp(THOROUGH_TEN)).isEqualTo(INITIAL_TIMESTAMP); progress.updateNumberOfShards(64); progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 32L); progress.updateLastSweptTimestamp(THOROUGH_TEN, 128L); assertThat(progress.getNumberOfShards()).isEqualTo(64); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(32L); assertThat(progress.getLastSweptTimestamp(THOROUGH_TEN)).isEqualTo(128L); }
@Test public void canUpdateSweptTimestamp() { progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 1024L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(1024L); }
@Test public void deletesGetBatched() { TargetedSweeper sweeperConservative = getSingleShardSweeper(); int numberOfTimestamps = 5 * BATCH_SIZE_KVS / MAX_CELLS_GENERIC + 1; commitTransactionsWithWritesIntoUniqueCells(numberOfTimestamps, MAX_CELLS_GENERIC, sweeperConservative); sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, times(6)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getAllValues().stream().map(Map::size).mapToInt(x -> x).sum()) .isEqualTo(5 * BATCH_SIZE_KVS + MAX_CELLS_GENERIC); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(maxTsForFinePartition(0)); }
@Test public void attemptingToDecreaseSweptTimestampIsNoop() { progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 1024L); progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 512L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(1024L); }
@Test public void batchIncludesAllWritesWithTheSameTimestampAndDoesNotSkipOrRepeatAnyWritesInNextIteration() { TargetedSweeper sweeperConservative = getSingleShardSweeper(); int relativePrime = MAX_CELLS_GENERIC - 1; // this assertion verifies that the test checks what we want. If it fails, change the value of relativePrime assertThat(SWEEP_BATCH_SIZE % relativePrime).isNotEqualTo(0); int minTsToReachBatchSize = (SWEEP_BATCH_SIZE - 1) / relativePrime + 1; commitTransactionsWithWritesIntoUniqueCells(minTsToReachBatchSize + 1, relativePrime, sweeperConservative); // first iteration of sweep should include all writes corresponding to timestamp 999 + minCellsToReachBatchSize, // since deletes are batched, we do not specify the number of calls to delete sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, atLeast(1)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getAllValues().stream().map(Map::size).mapToInt(x -> x).sum()) .isEqualTo(minTsToReachBatchSize * relativePrime); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(1000 + minTsToReachBatchSize - 1); //second iteration should only contain writes corresponding to timestamp 1000 + minCellsToReachBatchSize sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); verify(spiedKvs, atLeast(2)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getValue().size()).isEqualTo(relativePrime); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(maxTsForFinePartition(0)); }
@Test public void doNotSweepAnythingAfterEntryWithCommitTsAfterSweepTs() { immutableTs = 1000L; // put 4 writes committed at the same timestamp as start timestamp, and put one committed at sweep timestamp enqueueWriteCommitted(TABLE_CONS, 900); enqueueWriteCommitted(TABLE_CONS, 910); enqueueWriteCommitted(TABLE_CONS, 920); enqueueWriteCommitedAt(TABLE_CONS, 950, immutableTs); enqueueWriteCommitted(TABLE_CONS, 970); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(920L); ArgumentCaptor<Map> argument = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, times(1)).deleteAllTimestamps(eq(TABLE_CONS), argument.capture(), eq(false)); assertThat(argument.getValue()).containsValue(920L); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(920L); verify(spiedKvs, times(1)).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); immutableTs = 1001L; sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(1001L - 1L); // we have now had a total of 2 calls to deleteAllTimestamps, 1 from before and one new verify(spiedKvs, times(2)).deleteAllTimestamps(eq(TABLE_CONS), argument.capture(), eq(false)); assertThat(argument.getValue()).containsValue(970L); }
@Test public void doNotDeleteAnythingAfterEntryWithCommitTsAfterSweepTs() { immutableTs = 1000L; enqueueWriteUncommitted(TABLE_CONS, 900); enqueueWriteUncommitted(TABLE_CONS, 920); enqueueWriteCommitedAt(TABLE_CONS, 950, 2000); enqueueWriteUncommitted(TABLE_CONS, 970); enqueueWriteUncommitted(TABLE_CONS, 1110); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(920L); verify(spiedKvs, never()).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); ArgumentCaptor<Multimap> multimap = ArgumentCaptor.forClass(Multimap.class); verify(spiedKvs, times(1)).delete(eq(TABLE_CONS), multimap.capture()); assertThat(multimap.getValue().keySet()).containsExactly(DEFAULT_CELL); assertThat(multimap.getValue().values()).containsExactly(900L, 920L); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(920L); verify(spiedKvs, never()).deleteAllTimestamps(any(TableReference.class), anyMap(), eq(false)); verify(spiedKvs, times(1)).delete(any(TableReference.class), any(Multimap.class)); assertReadAtTimestampReturnsValue(TABLE_CONS, 1500L, 1110L); immutableTs = 2009L; sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(CONS_SHARD))).isEqualTo(2009L - 1L); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, times(1)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getValue()).containsValue(950L); assertReadAtTimestampReturnsValue(TABLE_CONS, 1500L, 950L); }
@Test public void doNotMissSingleWriteInNextIteration() { TargetedSweeper sweeperConservative = getSingleShardSweeper(); int minTsToReachBatchSize = (SWEEP_BATCH_SIZE - 1) / MAX_CELLS_GENERIC + 1; commitTransactionsWithWritesIntoUniqueCells(minTsToReachBatchSize, MAX_CELLS_GENERIC, sweeperConservative); // put one additional transaction with a single write after putTimestampIntoTransactionTable(1000 + minTsToReachBatchSize, 1000 + minTsToReachBatchSize); Cell cell = Cell.create(PtBytes.toBytes(1000 + minTsToReachBatchSize), PtBytes.toBytes(0)); sweeperConservative.enqueue(ImmutableList.of(WriteInfo.write(TABLE_CONS, cell, 1000 + minTsToReachBatchSize))); // first iteration of sweep should include all but one of the writes, since deletes are batched, we do not // specify the number of calls to delete sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, atLeast(1)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getAllValues().stream().map(Map::size).mapToInt(x -> x).sum()) .isEqualTo(SWEEP_BATCH_SIZE); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(1000 + minTsToReachBatchSize - 1); // second iteration of sweep should contain the remaining write sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); verify(spiedKvs, atLeast(2)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getValue().size()).isEqualTo(1); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(maxTsForFinePartition(0)); }
private Optional<Long> readThorough(int shardNumber) { return sweepableTimestamps.nextSweepableTimestampPartition( thorough(shardNumber), progress.getLastSweptTimestamp(ShardAndStrategy.thorough(shardNumber)), Sweeper.THOROUGH.getSweepTimestamp(timestampsSupplier)); }
private Optional<Long> readConservative(int shardNumber) { return sweepableTimestamps.nextSweepableTimestampPartition( conservative(shardNumber), progress.getLastSweptTimestamp(ShardAndStrategy.conservative(shardNumber)), Sweeper.CONSERVATIVE.getSweepTimestamp(timestampsSupplier)); }
long lastSweptTs = progress.getLastSweptTimestamp(shardStrategy);
/** * Updates the persisted last swept timestamp for the given shard and strategy to timestamp if it is greater than * the currently persisted last swept timestamp. * * @param shardAndStrategy shard and strategy to update for * @param timestamp timestamp to update to * @return the latest known persisted sweep timestamp for the shard and strategy */ public long updateLastSweptTimestamp(ShardAndStrategy shardAndStrategy, long timestamp) { return increaseValueFromToAtLeast(shardAndStrategy, getLastSweptTimestamp(shardAndStrategy), timestamp); }
long lastSweptTs = progress.getLastSweptTimestamp(shardStrategy);