deleter.sweep(sweepBatch.writes(), Sweeper.of(shardStrategy)); if (!sweepBatch.isEmpty()) { log.debug("Put {} ranged tombstones and swept up to timestamp {} for {}.", SafeArg.of("tombstones", sweepBatch.writes().size()), SafeArg.of("lastSweptTs", sweepBatch.lastSweptTimestamp()), SafeArg.of("shardStrategy", shardStrategy.toText())); cleaner.clean(shardStrategy, lastSweptTs, sweepBatch.lastSweptTimestamp(), sweepBatch.dedicatedRows()); metrics.updateNumberOfTombstones(shardStrategy, sweepBatch.writes().size()); metrics.updateProgressForShard(shardStrategy, sweepBatch.lastSweptTimestamp()); if (sweepBatch.isEmpty()) { metrics.registerOccurrenceOf(SweepOutcome.NOTHING_TO_SWEEP); } else {
SweepBatch getNextBatchToSweep(ShardAndStrategy shardStrategy, long lastSweptTs, long sweepTs) { return sweepableTimestamps.nextSweepableTimestampPartition(shardStrategy, lastSweptTs, sweepTs) .map(fine -> sweepableCells.getBatchForPartition(shardStrategy, fine, lastSweptTs, sweepTs)) .orElse(SweepBatch.of(ImmutableList.of(), DedicatedRows.of(ImmutableList.of()), sweepTs - 1L)); } }
@Test public void readOnlyMostRecentTimestampForRange() { writeToDefaultCellCommitted(sweepableCells, TS - 1, TABLE_CONS); writeToDefaultCellCommitted(sweepableCells, TS + 2, TABLE_CONS); writeToDefaultCellCommitted(sweepableCells, TS - 2, TABLE_CONS); writeToDefaultCellCommitted(sweepableCells, TS + 1, TABLE_CONS); SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 3, TS); assertThat(conservativeBatch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS - 1)); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(TS - 1); conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 3, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS + 2)); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(SMALL_SWEEP_TS - 1); }
default boolean isEmpty() { return writes().isEmpty(); }
@Test public void lastSweptTimestampIsMinimumOfSweepTsAndEndOfFinePartitionWhenThereAreMatches() { SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(SMALL_SWEEP_TS - 1); conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 1, Long.MAX_VALUE); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(endOfFinePartitionForTs(TS)); }
@Test public void getBatchReturnsDedicatedRowsSeen() { useSingleShard(); long timestamp = TS + 1; writeCommittedConservativeRowForTimestamp(timestamp, MAX_CELLS_DEDICATED * 2 + 1); DedicatedRows expectedDedicatedRows = DedicatedRows.of(LongStream.range(0, 3) .mapToObj(rowNumber -> ImmutableTargetedSweepMetadata.builder() .conservative(true) .dedicatedRow(true) .dedicatedRowNumber(rowNumber) .shard(0) .build() .persistToBytes()) .map(metadata -> SweepableCellsRow.of(timestamp, metadata)) .collect(Collectors.toList())); assertThat(readConservative(0, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS).dedicatedRows()) .isEqualTo(expectedDedicatedRows); }
@Test public void returnWhenMoreThanSweepBatchSize() { useSingleShard(); long iterationWrites = 1 + SWEEP_BATCH_SIZE / 5; for (int i = 1; i <= 10; i++) { writeCommittedConservativeRowForTimestamp(i, iterationWrites); } SweepBatch conservativeBatch = readConservative(0, 0L, -1L, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes().size()).isEqualTo(SWEEP_BATCH_SIZE + 5); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(5); SweepMetricsAssert.assertThat(metricsManager).hasEnqueuedWritesConservativeEqualTo( 10 * iterationWrites + 1); SweepMetricsAssert.assertThat(metricsManager).hasEntriesReadConservativeEqualTo( 5 * iterationWrites); SweepMetricsAssert.assertThat(metricsManager).hasAbortedWritesDeletedConservativeEquals(0); }
@Test public void cannotReadEntryOutOfRange() { SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).isEmpty(); conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, 0L, TS); assertThat(conservativeBatch.writes()).isEmpty(); }
@Test public void lastSweptTimestampIsMinimumOfSweepTsAndEndOfFinePartitionWhenNoMatches() { SweepBatch conservativeBatch = readConservative(shardCons + 1, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(SMALL_SWEEP_TS - 1); conservativeBatch = readConservative(shardCons + 1, TS_FINE_PARTITION, TS - 1, Long.MAX_VALUE); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(endOfFinePartitionForTs(TS)); }
deleter.sweep(sweepBatch.writes(), Sweeper.of(shardStrategy)); if (!sweepBatch.isEmpty()) { log.debug("Put {} ranged tombstones and swept up to timestamp {} for {}.", SafeArg.of("tombstones", sweepBatch.writes().size()), SafeArg.of("lastSweptTs", sweepBatch.lastSweptTimestamp()), SafeArg.of("shardStrategy", shardStrategy.toText())); cleaner.clean(shardStrategy, lastSweptTs, sweepBatch.lastSweptTimestamp(), sweepBatch.dedicatedRows()); metrics.updateNumberOfTombstones(shardStrategy, sweepBatch.writes().size()); metrics.updateProgressForShard(shardStrategy, sweepBatch.lastSweptTimestamp()); if (sweepBatch.isEmpty()) { metrics.registerOccurrenceOf(SweepOutcome.NOTHING_TO_SWEEP); } else {
@Test public void returnWhenMoreThanSweepBatchSizeWithRepeatsHasFewerEntries() { useSingleShard(); int iterationWrites = 1 + SWEEP_BATCH_SIZE / 5; for (int i = 1; i <= 10; i++) { writeCommittedConservativeRowZero(i, iterationWrites); } SweepBatch conservativeBatch = readConservative(0, 0L, -1L, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes().size()).isEqualTo(iterationWrites); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(5); SweepMetricsAssert.assertThat(metricsManager).hasEnqueuedWritesConservativeEqualTo( 10 * iterationWrites + 1); SweepMetricsAssert.assertThat(metricsManager).hasEntriesReadConservativeEqualTo( 5 * iterationWrites); SweepMetricsAssert.assertThat(metricsManager).hasAbortedWritesDeletedConservativeEquals(0); }
@Test public void cannotReadEntryForWrongPartition() { SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION - 1, 0L, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).isEmpty(); conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION + 1, TS - 1, Long.MAX_VALUE); assertThat(conservativeBatch.writes()).isEmpty(); }
SweepBatch getBatchForPartition(ShardAndStrategy shardStrategy, long partitionFine, long minTsExclusive, long sweepTs) { SweepableCellsRow row = computeRow(partitionFine, shardStrategy); RowColumnRangeIterator resultIterator = getRowColumnRange(row, partitionFine, minTsExclusive, sweepTs); PeekingIterator<Map.Entry<Cell, Value>> peekingResultIterator = Iterators.peekingIterator(resultIterator); WriteBatch writeBatch = getBatchOfWrites(row, peekingResultIterator, sweepTs); Multimap<Long, WriteInfo> writesByStartTs = writeBatch.writesByStartTs; maybeMetrics.ifPresent(metrics -> metrics.updateEntriesRead(shardStrategy, writesByStartTs.size())); log.debug("Read {} entries from the sweep queue.", SafeArg.of("number", writesByStartTs.size())); TimestampsToSweep tsToSweep = getTimestampsToSweepDescendingAndCleanupAborted( shardStrategy, minTsExclusive, sweepTs, writesByStartTs); Collection<WriteInfo> writes = getWritesToSweep(writesByStartTs, tsToSweep.timestampsDescending()); DedicatedRows filteredDedicatedRows = getDedicatedRowsToClear(writeBatch.dedicatedRows, tsToSweep); long lastSweptTs = getLastSweptTs(tsToSweep, peekingResultIterator, partitionFine, sweepTs); return SweepBatch.of(writes, filteredDedicatedRows, lastSweptTs); }
@Test public void canReadMultipleEntriesInSingleShardDifferentTransactions() { writeToCellCommitted(sweepableCells, TS, getCellWithFixedHash(1), TABLE_CONS); writeToCellCommitted(sweepableCells, TS + 1, getCellWithFixedHash(2), TABLE_CONS); SweepBatch conservativeBatch = readConservative(FIXED_SHARD, TS_FINE_PARTITION, TS - 1, TS + 2); assertThat(conservativeBatch.writes()).containsExactlyInAnyOrder( WriteInfo.write(TABLE_CONS, getCellWithFixedHash(1), TS), WriteInfo.write(TABLE_CONS, getCellWithFixedHash(2), TS + 1)); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(TS + 1); }
@Test public void canReadMultipleEntriesInSingleShardSameTransactionMultipleDedicated() { useSingleShard(); List<WriteInfo> writes = writeCommittedConservativeRowForTimestamp(TS + 1, MAX_CELLS_DEDICATED + 1); SweepBatch conservativeBatch = readConservative(0, TS_FINE_PARTITION, TS, TS + 2); assertThat(conservativeBatch.writes().size()).isEqualTo(writes.size()); assertThat(conservativeBatch.writes()).contains(writes.get(0), writes.get(writes.size() - 1)); }
SweepBatch getNextBatchToSweep(ShardAndStrategy shardStrategy, long lastSweptTs, long sweepTs) { return sweepableTimestamps.nextSweepableTimestampPartition(shardStrategy, lastSweptTs, sweepTs) .map(fine -> sweepableCells.getBatchForPartition(shardStrategy, fine, lastSweptTs, sweepTs)) .orElse(SweepBatch.of(ImmutableList.of(), DedicatedRows.of(ImmutableList.of()), sweepTs - 1L)); } }
@Test public void returnNothingWhenMoreThanSweepBatchUncommitted() { useSingleShard(); int iterationWrites = 1 + SWEEP_BATCH_SIZE / 5; for (int i = 1; i <= 10; i++) { writeWithoutCommitConservative(i, i, iterationWrites); } writeCommittedConservativeRowForTimestamp(10, iterationWrites); SweepBatch conservativeBatch = readConservative(0, 0L, -1L, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).isEmpty(); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(5); SweepMetricsAssert.assertThat(metricsManager).hasEnqueuedWritesConservativeEqualTo( 11 * iterationWrites + 1); SweepMetricsAssert.assertThat(metricsManager).hasEntriesReadConservativeEqualTo( 5 * iterationWrites); SweepMetricsAssert.assertThat(metricsManager).hasAbortedWritesDeletedConservativeEquals( 5 * iterationWrites); }
@Test public void cannotReadEntryForWrongShard() { SweepBatch conservativeBatch = readConservative(shardCons + 1, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).isEmpty(); }
SweepBatch getBatchForPartition(ShardAndStrategy shardStrategy, long partitionFine, long minTsExclusive, long sweepTs) { SweepableCellsRow row = computeRow(partitionFine, shardStrategy); RowColumnRangeIterator resultIterator = getRowColumnRange(row, partitionFine, minTsExclusive, sweepTs); PeekingIterator<Map.Entry<Cell, Value>> peekingResultIterator = Iterators.peekingIterator(resultIterator); WriteBatch writeBatch = getBatchOfWrites(row, peekingResultIterator, sweepTs); Multimap<Long, WriteInfo> writesByStartTs = writeBatch.writesByStartTs; maybeMetrics.ifPresent(metrics -> metrics.updateEntriesRead(shardStrategy, writesByStartTs.size())); log.debug("Read {} entries from the sweep queue.", SafeArg.of("number", writesByStartTs.size())); TimestampsToSweep tsToSweep = getTimestampsToSweepDescendingAndCleanupAborted( shardStrategy, minTsExclusive, sweepTs, writesByStartTs); Collection<WriteInfo> writes = getWritesToSweep(writesByStartTs, tsToSweep.timestampsDescending()); DedicatedRows filteredDedicatedRows = getDedicatedRowsToClear(writeBatch.dedicatedRows, tsToSweep); long lastSweptTs = getLastSweptTs(tsToSweep, peekingResultIterator, partitionFine, sweepTs); return SweepBatch.of(writes, filteredDedicatedRows, lastSweptTs); }
@Test public void canReadMultipleEntriesInSingleShardSameTransactionOneDedicated() { List<WriteInfo> writes = writeToCellsInFixedShard(sweepableCells, TS, MAX_CELLS_GENERIC * 2 + 1, TABLE_CONS); SweepBatch conservativeBatch = readConservative(FIXED_SHARD, TS_FINE_PARTITION, TS - 1, TS + 1); assertThat(conservativeBatch.writes().size()).isEqualTo(MAX_CELLS_GENERIC * 2 + 1); assertThat(conservativeBatch.writes()).hasSameElementsAs(writes); }