private void enqueueWriteCommitted(TableReference tableRef, long ts) { putTimestampIntoTransactionTable(ts, ts); enqueueWriteUncommitted(tableRef, ts); }
private void enqueueWriteCommitedAt(TableReference tableRef, long startTs, long commitTs) { putTimestampIntoTransactionTable(startTs, commitTs); enqueueWriteUncommitted(tableRef, startTs); }
private void enqueueTombstone(TableReference tableRef, long ts) { putTimestampIntoTransactionTable(ts, ts); sweepQueue.enqueue(tombstoneToDefaultCell(tableRef, ts), ts); }
private void commitTransactionsWithWritesIntoUniqueCells(int transactions, int writes, TargetedSweeper sweeper) { for (int i = 1000; i < 1000 + transactions; i++) { putTimestampIntoTransactionTable(i, i); List<WriteInfo> writeInfos = new ArrayList<>(); for (int j = 0; j < writes; j++) { Cell cell = Cell.create(PtBytes.toBytes(i), PtBytes.toBytes(j)); writeInfos.add(WriteInfo.write(TABLE_CONS, cell, i)); } sweeper.enqueue(writeInfos); } }
@Test public void doNotMissSingleWriteInNextIteration() { TargetedSweeper sweeperConservative = getSingleShardSweeper(); int minTsToReachBatchSize = (SWEEP_BATCH_SIZE - 1) / MAX_CELLS_GENERIC + 1; commitTransactionsWithWritesIntoUniqueCells(minTsToReachBatchSize, MAX_CELLS_GENERIC, sweeperConservative); // put one additional transaction with a single write after putTimestampIntoTransactionTable(1000 + minTsToReachBatchSize, 1000 + minTsToReachBatchSize); Cell cell = Cell.create(PtBytes.toBytes(1000 + minTsToReachBatchSize), PtBytes.toBytes(0)); sweeperConservative.enqueue(ImmutableList.of(WriteInfo.write(TABLE_CONS, cell, 1000 + minTsToReachBatchSize))); // first iteration of sweep should include all but one of the writes, since deletes are batched, we do not // specify the number of calls to delete sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, atLeast(1)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getAllValues().stream().map(Map::size).mapToInt(x -> x).sum()) .isEqualTo(SWEEP_BATCH_SIZE); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(1000 + minTsToReachBatchSize - 1); // second iteration of sweep should contain the remaining write sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); verify(spiedKvs, atLeast(2)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getValue().size()).isEqualTo(1); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(maxTsForFinePartition(0)); }
@Test public void stopReadingEarlyInOtherShardWhenEncounteringEntryKnownToBeCommittedAfterSweepTs() { immutableTs = 100L; putTimestampIntoTransactionTable(50, 200); Map<Integer, Integer> largeWriteDistribution = enqueueAtLeastThresholdWritesInDefaultShardWithStartTs(100, 50); int writesInDedicated = largeWriteDistribution.get(CONS_SHARD); int otherShard = IntMath.mod(CONS_SHARD + 1, DEFAULT_SHARDS); int writesInOther = largeWriteDistribution.get(otherShard); assertThat(writesInOther).isGreaterThan(0); // first iteration reads all before giving up sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasEntriesReadConservativeEqualTo(writesInDedicated); // we read a reference to bad entries and give up sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(otherShard)); assertThat(metricsManager).hasEntriesReadConservativeEqualTo(writesInDedicated + 1); immutableTs = 250; // we now read all to the end sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(otherShard)); assertThat(metricsManager).hasEntriesReadConservativeEqualTo(writesInDedicated + 1 + writesInOther); }
@Test public void stopReadingEarlyWhenEncounteringEntryKnownToBeCommittedAfterSweepTs() { immutableTs = 100L; enqueueWriteCommitted(TABLE_CONS, 10); enqueueWriteCommitedAt(TABLE_CONS, 30, 150); putTimestampIntoTransactionTable(50, 200); Map<Integer, Integer> largeWriteDistribution = enqueueAtLeastThresholdWritesInDefaultShardWithStartTs(100, 50); int writesInDedicated = largeWriteDistribution.get(CONS_SHARD); enqueueWriteUncommitted(TABLE_CONS, 70); enqueueWriteCommitted(TABLE_CONS, 90); // first iteration reads all before giving up sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasEntriesReadConservativeEqualTo(4 + writesInDedicated); // we read one entry and give up sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasEntriesReadConservativeEqualTo(4 + writesInDedicated + 1); immutableTs = 170; // we read one good entry and then a reference to bad entries and give up sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasEntriesReadConservativeEqualTo(4 + writesInDedicated + 3); immutableTs = 250; // we now read all to the end sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertThat(metricsManager).hasEntriesReadConservativeEqualTo(4 + writesInDedicated + 3 + writesInDedicated + 2); }
@Test public void sweepableCellsGetsScrubbedWheneverLastSweptInNewPartition() { long tsSecondPartitionFine = LOW_TS + TS_FINE_GRANULARITY; enqueueWriteCommitted(TABLE_CONS, LOW_TS); enqueueWriteCommitted(TABLE_CONS, LOW_TS + 1L); enqueueAtLeastThresholdWritesInDefaultShardWithStartTs(100, LOW_TS + 2L); putTimestampIntoTransactionTable(LOW_TS + 2L, LOW_TS + 2L); enqueueWriteCommitted(TABLE_CONS, tsSecondPartitionFine); enqueueWriteCommitted(TABLE_CONS, getSweepTsCons()); // last swept timestamp: TS_FINE_GRANULARITY - 1 sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertSweepableCellsHasEntryForTimestamp(LOW_TS + 1); assertSweepableCellsHasEntryForTimestamp(tsSecondPartitionFine); assertSweepableCellsHasEntryForTimestamp(getSweepTsCons()); // last swept timestamp: 2 * TS_FINE_GRANULARITY - 1 sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertSweepableCellsHasNoEntriesInPartitionOfTimestamp(LOW_TS + 1); assertSweepableCellsHasEntryForTimestamp(tsSecondPartitionFine); assertSweepableCellsHasEntryForTimestamp(getSweepTsCons()); // last swept timestamp: largestBeforeSweepTs sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(CONS_SHARD)); assertSweepableCellsHasNoEntriesInPartitionOfTimestamp(LOW_TS + 1); assertSweepableCellsHasNoEntriesInPartitionOfTimestamp(tsSecondPartitionFine); assertSweepableCellsHasEntryForTimestamp(getSweepTsCons()); assertSweepableCellsHasNoDedicatedRowsForShard(CONS_SHARD); }