@Before public void setup() { super.setup(); sweepQueue = TargetedSweeper.createUninitializedForTest(() -> 1); sweepQueue.initializeWithoutRunning( timestampsSupplier, mock(TimelockService.class), kvs, txService, mock(TargetedSweepFollower.class)); }
public void runIterationOfTargetedSweep() { targetedSweeper.get().sweepNextBatch(ShardAndStrategy.conservative(0)); targetedSweeper.get().sweepNextBatch(ShardAndStrategy.thorough(0)); }
private void createAndInitializeSweepersAndWaitForOneBackgroundIteration(int sweepers, int shards, int threads, TimelockService stickyLockService) throws InterruptedException { for (int i = 0; i < sweepers; i++) { TargetedSweeper sweeperInstance = TargetedSweeper .createUninitialized(metricsManager, () -> true, () -> shards, threads, 0, ImmutableList.of()); sweeperInstance.initializeWithoutRunning( timestampsSupplier, stickyLockService, spiedKvs, txnService, mockFollower); sweeperInstance.runInBackground(); } waitUntilBackgroundSweepRunsOneIteration(); }
@Override public void initialize(TransactionManager txManager) { initializeWithoutRunning(txManager); runInBackground(); }
@Test public void callingEnqueueAndSweepOnUninitializedSweeperThrows() { TargetedSweeper uninitializedSweeper = TargetedSweeper.createUninitializedForTest(null); assertThatThrownBy(() -> uninitializedSweeper.enqueue(ImmutableList.of())) .isInstanceOf(NotInitializedException.class) .hasMessageContaining("Targeted Sweeper"); assertThatThrownBy(() -> uninitializedSweeper.sweepNextBatch(ShardAndStrategy.conservative(0))) .isInstanceOf(NotInitializedException.class) .hasMessageContaining("Targeted Sweeper"); }
@Override protected MultiTableSweepQueueWriter getSweepQueueWriterUninitialized() { return TargetedSweeper.createUninitializedForTest(() -> 128); }
when(mockServices.getTransactionService()).thenReturn(transactionService); when(mockServices.getKeyValueService()).thenReturn(kvs); TargetedSweeper sweeper = TargetedSweeper.createUninitializedForTest(() -> 1); SerializableTransactionManager txManager = SerializableTransactionManager.createForTest( MetricsManagers.createForTests(), 4, sweeper); sweeper.initialize(txManager); when(mockServices.getTransactionManager()).thenReturn(txManager); return mockServices;
@Override protected void put(final TableReference tableRef, Cell cell, final String val, final long ts) { super.put(tableRef, cell, val, ts); sweepQueue.enqueue(ImmutableMap.of(tableRef, ImmutableMap.of(cell, PtBytes.toBytes(val))), ts); }
@Test public void doNotMissSingleWriteInNextIteration() { TargetedSweeper sweeperConservative = getSingleShardSweeper(); int minTsToReachBatchSize = (SWEEP_BATCH_SIZE - 1) / MAX_CELLS_GENERIC + 1; commitTransactionsWithWritesIntoUniqueCells(minTsToReachBatchSize, MAX_CELLS_GENERIC, sweeperConservative); // put one additional transaction with a single write after putTimestampIntoTransactionTable(1000 + minTsToReachBatchSize, 1000 + minTsToReachBatchSize); Cell cell = Cell.create(PtBytes.toBytes(1000 + minTsToReachBatchSize), PtBytes.toBytes(0)); sweeperConservative.enqueue(ImmutableList.of(WriteInfo.write(TABLE_CONS, cell, 1000 + minTsToReachBatchSize))); // first iteration of sweep should include all but one of the writes, since deletes are batched, we do not // specify the number of calls to delete sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, atLeast(1)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getAllValues().stream().map(Map::size).mapToInt(x -> x).sum()) .isEqualTo(SWEEP_BATCH_SIZE); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(1000 + minTsToReachBatchSize - 1); // second iteration of sweep should contain the remaining write sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); verify(spiedKvs, atLeast(2)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getValue().size()).isEqualTo(1); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(maxTsForFinePartition(0)); }
@VisibleForTesting static TargetedSweeper createUninitializedForTest(MetricsManager metricsManager, Supplier<Boolean> enabled, Supplier<Integer> shards) { return createUninitialized(metricsManager, enabled, shards, 0, 0, ImmutableList.of()); }
public void initializeWithoutRunning(TransactionManager txManager) { initializeWithoutRunning(SpecialTimestampsSupplier.create(txManager), txManager.getTimelockService(), txManager.getKeyValueService(), TransactionServices.createTransactionService(txManager.getKeyValueService(), CoordinationServices.createDefault( txManager.getKeyValueService(), txManager.getTimestampService(), false)), new TargetedSweepFollower(followers, txManager)); }
/** * Creates a targeted sweeper, without initializing any of the necessary resources. You must call the * {@link #initializeWithoutRunning(SpecialTimestampsSupplier, TimelockService, KeyValueService, * TransactionService, TargetedSweepFollower)} method before any writes can be made to the sweep queue, or before * the background sweep job can run. * * @param enabled live reloadable config controlling whether background threads should perform targeted sweep. * @param shardsConfig live reloadable config specifying the desired number of shards. Since the number of shards * must never be reduced, this will be ignored if the persisted number of shards is greater. * @param conservativeThreads number of conservative threads to use for background targeted sweep. * @param thoroughThreads number of thorough threads to use for background targeted sweep. * @param followers follower used for sweeps, as defined by your schema. * @return returns an uninitialized targeted sweeper */ public static TargetedSweeper createUninitialized(MetricsManager metrics, Supplier<Boolean> enabled, Supplier<Integer> shardsConfig, int conservativeThreads, int thoroughThreads, List<Follower> followers) { return new TargetedSweeper(metrics, enabled, shardsConfig, conservativeThreads, thoroughThreads, followers); }
public static TargetedSweeper createUninitializedForTest(Supplier<Integer> shards) { return createUninitializedForTest(MetricsManagers.createForTests(), () -> true, shards); }
private TargetedSweeper initializeAndGet(TargetedSweeper sweeper, TransactionManager txManager) { sweeper.initializeWithoutRunning( new SpecialTimestampsSupplier(txManager::getImmutableTimestamp, txManager::getImmutableTimestamp), txManager.getTimelockService(), txManager.getKeyValueService(), TransactionServices.createForTesting( txManager.getKeyValueService(), txManager.getTimestampService(), false), new TargetedSweepFollower(ImmutableList.of(FOLLOWER), txManager)); sweeper.runInBackground(); return sweeper; }
@Before public void setUp() throws Exception { timestampService = new InMemoryTimestampService(); KeyValueService kvs = getBaseKeyValueService(); keyValueServiceWithStats = new StatsTrackingKeyValueService(kvs); keyValueService = spy(new TrackingKeyValueService(keyValueServiceWithStats)); TransactionTables.createTables(kvs); transactionService = TransactionServices.createForTesting(keyValueService, timestampService, false); conflictDetectionManager = ConflictDetectionManagers.createWithoutWarmingCache(keyValueService); sweepStrategyManager = SweepStrategyManagers.createDefault(keyValueService); sweepQueue = spy(TargetedSweeper.createUninitializedForTest(() -> sweepQueueShards)); serializableTxManager = new TestTransactionManagerImpl( metricsManager, keyValueService, timestampService, timestampService, lockClient, lockService, transactionService, conflictDetectionManager, sweepStrategyManager, sweepQueue, MoreExecutors.newDirectExecutorService()); sweepQueue.initialize(serializableTxManager); txManager = new CachingTestTransactionManager(serializableTxManager); }
private List<WriteInfo> getEnqueuedWritesNumber(int index) { ArgumentCaptor<List> writes = ArgumentCaptor.forClass(List.class); verify(sweepQueue, atLeast(index)).enqueue(writes.capture()); return writes.getAllValues().get(index - 1); }
private MultiTableSweepQueueWriter uninitializedTargetedSweeper( MetricsManager metricsManager, TargetedSweepInstallConfig config, Follower follower, Supplier<TargetedSweepRuntimeConfig> runtime) { if (!config.enableSweepQueueWrites()) { return MultiTableSweepQueueWriter.NO_OP; } return TargetedSweeper.createUninitialized( metricsManager, Suppliers.compose(TargetedSweepRuntimeConfig::enabled, runtime::get), Suppliers.compose(TargetedSweepRuntimeConfig::shards, runtime::get), config.conservativeThreads(), config.thoroughThreads(), ImmutableList.of(follower)); }
public void initializeWithoutRunning(TransactionManager txManager) { initializeWithoutRunning(SpecialTimestampsSupplier.create(txManager), txManager.getTimelockService(), txManager.getKeyValueService(), TransactionServices.createTransactionService(txManager.getKeyValueService(), CoordinationServices.createDefault( txManager.getKeyValueService(), txManager.getTimestampService(), false)), new TargetedSweepFollower(followers, txManager)); }
/** * Creates a targeted sweeper, without initializing any of the necessary resources. You must call the * {@link #initializeWithoutRunning(SpecialTimestampsSupplier, TimelockService, KeyValueService, * TransactionService, TargetedSweepFollower)} method before any writes can be made to the sweep queue, or before * the background sweep job can run. * * @param enabled live reloadable config controlling whether background threads should perform targeted sweep. * @param shardsConfig live reloadable config specifying the desired number of shards. Since the number of shards * must never be reduced, this will be ignored if the persisted number of shards is greater. * @param conservativeThreads number of conservative threads to use for background targeted sweep. * @param thoroughThreads number of thorough threads to use for background targeted sweep. * @param followers follower used for sweeps, as defined by your schema. * @return returns an uninitialized targeted sweeper */ public static TargetedSweeper createUninitialized(MetricsManager metrics, Supplier<Boolean> enabled, Supplier<Integer> shardsConfig, int conservativeThreads, int thoroughThreads, List<Follower> followers) { return new TargetedSweeper(metrics, enabled, shardsConfig, conservativeThreads, thoroughThreads, followers); }
@Override protected Optional<SweepResults> completeSweep(TableReference ignored, long ts) { when(timestampsSupplier.getUnreadableTimestamp()).thenReturn(ts); when(timestampsSupplier.getImmutableTimestamp()).thenReturn(ts); sweepQueue.sweepNextBatch(ShardAndStrategy.conservative(0)); sweepQueue.sweepNextBatch(ShardAndStrategy.thorough(0)); return Optional.empty(); }