public static SweepQueue create( TargetedSweepMetrics metrics, KeyValueService kvs, TimelockService timelock, Supplier<Integer> shardsConfig, TransactionService transaction, TargetedSweepFollower follower) { return new SweepQueue(SweepQueueFactory.create(metrics, kvs, timelock, shardsConfig, transaction), follower); }
/** * This method initializes all the resources necessary for the targeted sweeper. This method should only be called * once the kvs is ready. * @param timestamps supplier of unreadable and immutable timestamps. * @param timelockService TimeLockService to use for synchronizing iterations of sweep on different nodes * @param kvs key value service that must be already initialized. * @param transaction transaction service for checking if values were committed and rolling back if necessary * @param follower followers used for sweeps. */ public void initializeWithoutRunning( SpecialTimestampsSupplier timestamps, TimelockService timelockService, KeyValueService kvs, TransactionService transaction, TargetedSweepFollower follower) { if (isInitialized) { return; } Preconditions.checkState(kvs.isInitialized(), "Attempted to initialize targeted sweeper with an uninitialized backing KVS."); metrics = TargetedSweepMetrics.create(metricsManager, timelockService, kvs, SweepQueueUtils.REFRESH_TIME); queue = SweepQueue.create(metrics, kvs, timelockService, shardsConfig, transaction, follower); timestampsSupplier = timestamps; timeLock = timelockService; isInitialized = true; }
@Override public void enqueue(List<WriteInfo> writes) { assertInitialized(); queue.enqueue(writes); }
@Before public void setup() { kvs = new InMemoryKeyValueService(true); progress = spy(new ShardProgress(kvs)); numShardSupplier = SweepQueue.createProgressUpdatingSupplier(runtimeConfigSupplier, progress, 1); }
private int getShardAndIncrement() { return (int) (counter.getAndIncrement() % queue.getNumShards()); }
/** * Sweeps the next batch for the given shard and strategy. If the sweep is successful, we delete the processed * writes from the sweep queue and then update the sweep queue progress accordingly. * * @param shardStrategy shard and strategy to use */ @SuppressWarnings("checkstyle:RegexpMultiline") // Suppress VisibleForTesting warning @VisibleForTesting public void sweepNextBatch(ShardAndStrategy shardStrategy) { assertInitialized(); if (!runSweep.get()) { metrics.registerOccurrenceOf(SweepOutcome.DISABLED); return; } long maxTsExclusive = Sweeper.of(shardStrategy).getSweepTimestamp(timestampsSupplier); queue.sweepNextBatch(shardStrategy, maxTsExclusive); }
@Override protected MultiTableSweepQueueWriter getSweepQueueWriterInitialized() { return SweepQueue.createWriter(mock(TargetedSweepMetrics.class), keyValueService, timelockService, () -> 128); }
static SweepQueueFactory create( TargetedSweepMetrics metrics, KeyValueService kvs, TimelockService timelock, Supplier<Integer> shardsConfig, TransactionService transaction) { Schemas.createTablesAndIndexes(TargetedSweepSchema.INSTANCE.getLatestSchema(), kvs); ShardProgress shardProgress = new ShardProgress(kvs); Supplier<Integer> shards = createProgressUpdatingSupplier(shardsConfig, shardProgress, SweepQueueUtils.REFRESH_TIME); WriteInfoPartitioner partitioner = new WriteInfoPartitioner(kvs, shards); SweepableCells cells = new SweepableCells(kvs, partitioner, metrics, transaction); SweepableTimestamps timestamps = new SweepableTimestamps(kvs, partitioner); return new SweepQueueFactory(shardProgress, shards, cells, timestamps, metrics, kvs, timelock); }
private Optional<TargetedSweeperLock> tryToAcquireLockForNextShardAndStrategy() { return IntStream.range(0, queue.getNumShards()) .map(ignore -> getShardAndIncrement()) .mapToObj(shard -> TargetedSweeperLock.tryAcquire(shard, sweepStrategy, timeLock)) .filter(Optional::isPresent) .map(Optional::get) .findFirst(); }
/** * Sweeps the next batch for the given shard and strategy. If the sweep is successful, we delete the processed * writes from the sweep queue and then update the sweep queue progress accordingly. * * @param shardStrategy shard and strategy to use */ @SuppressWarnings("checkstyle:RegexpMultiline") // Suppress VisibleForTesting warning @VisibleForTesting public void sweepNextBatch(ShardAndStrategy shardStrategy) { assertInitialized(); if (!runSweep.get()) { metrics.registerOccurrenceOf(SweepOutcome.DISABLED); return; } long maxTsExclusive = Sweeper.of(shardStrategy).getSweepTimestamp(timestampsSupplier); queue.sweepNextBatch(shardStrategy, maxTsExclusive); }
static SweepQueueFactory create( TargetedSweepMetrics metrics, KeyValueService kvs, TimelockService timelock, Supplier<Integer> shardsConfig, TransactionService transaction) { Schemas.createTablesAndIndexes(TargetedSweepSchema.INSTANCE.getLatestSchema(), kvs); ShardProgress shardProgress = new ShardProgress(kvs); Supplier<Integer> shards = createProgressUpdatingSupplier(shardsConfig, shardProgress, SweepQueueUtils.REFRESH_TIME); WriteInfoPartitioner partitioner = new WriteInfoPartitioner(kvs, shards); SweepableCells cells = new SweepableCells(kvs, partitioner, metrics, transaction); SweepableTimestamps timestamps = new SweepableTimestamps(kvs, partitioner); return new SweepQueueFactory(shardProgress, shards, cells, timestamps, metrics, kvs, timelock); }
private int getShardAndIncrement() { return (int) (counter.getAndIncrement() % queue.getNumShards()); }
@Override public void enqueue(List<WriteInfo> writes) { assertInitialized(); queue.enqueue(writes); }
public static SweepQueue create( TargetedSweepMetrics metrics, KeyValueService kvs, TimelockService timelock, Supplier<Integer> shardsConfig, TransactionService transaction, TargetedSweepFollower follower) { return new SweepQueue(SweepQueueFactory.create(metrics, kvs, timelock, shardsConfig, transaction), follower); }
/** * This method initializes all the resources necessary for the targeted sweeper. This method should only be called * once the kvs is ready. * @param timestamps supplier of unreadable and immutable timestamps. * @param timelockService TimeLockService to use for synchronizing iterations of sweep on different nodes * @param kvs key value service that must be already initialized. * @param transaction transaction service for checking if values were committed and rolling back if necessary * @param follower followers used for sweeps. */ public void initializeWithoutRunning( SpecialTimestampsSupplier timestamps, TimelockService timelockService, KeyValueService kvs, TransactionService transaction, TargetedSweepFollower follower) { if (isInitialized) { return; } Preconditions.checkState(kvs.isInitialized(), "Attempted to initialize targeted sweeper with an uninitialized backing KVS."); metrics = TargetedSweepMetrics.create(metricsManager, timelockService, kvs, SweepQueueUtils.REFRESH_TIME); queue = SweepQueue.create(metrics, kvs, timelockService, shardsConfig, transaction, follower); timestampsSupplier = timestamps; timeLock = timelockService; isInitialized = true; }
private Optional<TargetedSweeperLock> tryToAcquireLockForNextShardAndStrategy() { return IntStream.range(0, queue.getNumShards()) .map(ignore -> getShardAndIncrement()) .mapToObj(shard -> TargetedSweeperLock.tryAcquire(shard, sweepStrategy, timeLock)) .filter(Optional::isPresent) .map(Optional::get) .findFirst(); }