@Test public void successfulLockAndUnlock() throws InterruptedException { LockToken lockToken = LockToken.of(UUID.randomUUID()); when(mockLockService.lock(any())) .thenReturn(() -> Optional.of(lockToken)); Optional<TargetedSweeperLock> maybeLock = TargetedSweeperLock .tryAcquire(1, TableMetadataPersistence.SweepStrategy.CONSERVATIVE, mockLockService); assertThat(maybeLock).isPresent(); TargetedSweeperLock lock = maybeLock.get(); assertThat(lock.getShardAndStrategy()).isEqualTo(ShardAndStrategy.conservative(1)); lock.unlock(); verify(mockLockService, times(1)).unlock(ImmutableSet.of(lockToken)); verify(mockLockService, times(1)).lock(any()); verifyNoMoreInteractions(mockLockService); }
public static Optional<TargetedSweeperLock> tryAcquire(int shard, TableMetadataPersistence.SweepStrategy strategy, TimelockService timeLock) { ShardAndStrategy shardStrategy = ShardAndStrategy.of(shard, strategy); LockDescriptor lock = StringLockDescriptor.of(shardStrategy.toText()); // We do not want the timeout to be too low to avoid a race condition where we give up too soon LockRequest request = LockRequest.of(ImmutableSet.of(lock), 100L); return timeLock.lock(request) .getTokenOrEmpty() .map(lockToken -> new TargetedSweeperLock(shardStrategy, timeLock, lockToken)); }
private void logUnlockException(Throwable th, Optional<TargetedSweeperLock> maybeLock) { if (maybeLock.isPresent()) { log.info("Failed to unlock targeted sweep lock for {}.", SafeArg.of("shardStrategy", maybeLock.get().getShardAndStrategy().toText()), th); } else { log.info("Failed to unlock targeted sweep lock for sweep strategy {}.", SafeArg.of("sweepStrategy", sweepStrategy), th); } }
private Optional<TargetedSweeperLock> tryToAcquireLockForNextShardAndStrategy() { return IntStream.range(0, queue.getNumShards()) .map(ignore -> getShardAndIncrement()) .mapToObj(shard -> TargetedSweeperLock.tryAcquire(shard, sweepStrategy, timeLock)) .filter(Optional::isPresent) .map(Optional::get) .findFirst(); }
private void logException(Throwable th, Optional<TargetedSweeperLock> maybeLock) { if (maybeLock.isPresent()) { log.warn("Targeted sweep for {} failed and will be retried later.", SafeArg.of("shardStrategy", maybeLock.get().getShardAndStrategy().toText()), th); } else { log.warn("Targeted sweep for sweep strategy {} failed and will be retried later.", SafeArg.of("sweepStrategy", sweepStrategy), th); } }
@Test public void unsuccessfulLock() throws InterruptedException { when(mockLockService.lock(any())).thenReturn(() -> Optional.empty()); Optional<TargetedSweeperLock> maybeLock = TargetedSweeperLock .tryAcquire(2, TableMetadataPersistence.SweepStrategy.THOROUGH, mockLockService); assertThat(maybeLock).isNotPresent(); verify(mockLockService, times(1)).lock(any()); verifyNoMoreInteractions(mockLockService); } }
private void runOneIteration() { Optional<TargetedSweeperLock> maybeLock = Optional.empty(); try { maybeLock = tryToAcquireLockForNextShardAndStrategy(); maybeLock.ifPresent(lock -> sweepNextBatch(lock.getShardAndStrategy())); } catch (InsufficientConsistencyException e) { metrics.registerOccurrenceOf(SweepOutcome.NOT_ENOUGH_DB_NODES_ONLINE); logException(e, maybeLock); } catch (Throwable th) { metrics.registerOccurrenceOf(SweepOutcome.ERROR); logException(th, maybeLock); } finally { try { maybeLock.ifPresent(TargetedSweeperLock::unlock); } catch (Throwable th) { logUnlockException(th, maybeLock); } } }
private Optional<TargetedSweeperLock> tryToAcquireLockForNextShardAndStrategy() { return IntStream.range(0, queue.getNumShards()) .map(ignore -> getShardAndIncrement()) .mapToObj(shard -> TargetedSweeperLock.tryAcquire(shard, sweepStrategy, timeLock)) .filter(Optional::isPresent) .map(Optional::get) .findFirst(); }
public static Optional<TargetedSweeperLock> tryAcquire(int shard, TableMetadataPersistence.SweepStrategy strategy, TimelockService timeLock) { ShardAndStrategy shardStrategy = ShardAndStrategy.of(shard, strategy); LockDescriptor lock = StringLockDescriptor.of(shardStrategy.toText()); // We do not want the timeout to be too low to avoid a race condition where we give up too soon LockRequest request = LockRequest.of(ImmutableSet.of(lock), 100L); return timeLock.lock(request) .getTokenOrEmpty() .map(lockToken -> new TargetedSweeperLock(shardStrategy, timeLock, lockToken)); }
private void logUnlockException(Throwable th, Optional<TargetedSweeperLock> maybeLock) { if (maybeLock.isPresent()) { log.info("Failed to unlock targeted sweep lock for {}.", SafeArg.of("shardStrategy", maybeLock.get().getShardAndStrategy().toText()), th); } else { log.info("Failed to unlock targeted sweep lock for sweep strategy {}.", SafeArg.of("sweepStrategy", sweepStrategy), th); } }
private void logException(Throwable th, Optional<TargetedSweeperLock> maybeLock) { if (maybeLock.isPresent()) { log.warn("Targeted sweep for {} failed and will be retried later.", SafeArg.of("shardStrategy", maybeLock.get().getShardAndStrategy().toText()), th); } else { log.warn("Targeted sweep for sweep strategy {} failed and will be retried later.", SafeArg.of("sweepStrategy", sweepStrategy), th); } }
private void runOneIteration() { Optional<TargetedSweeperLock> maybeLock = Optional.empty(); try { maybeLock = tryToAcquireLockForNextShardAndStrategy(); maybeLock.ifPresent(lock -> sweepNextBatch(lock.getShardAndStrategy())); } catch (InsufficientConsistencyException e) { metrics.registerOccurrenceOf(SweepOutcome.NOT_ENOUGH_DB_NODES_ONLINE); logException(e, maybeLock); } catch (Throwable th) { metrics.registerOccurrenceOf(SweepOutcome.ERROR); logException(th, maybeLock); } finally { try { maybeLock.ifPresent(TargetedSweeperLock::unlock); } catch (Throwable th) { logUnlockException(th, maybeLock); } } }