@Test public void extraSweepersGiveUpAfterFailingToAcquireEnoughTimes() throws InterruptedException { int shards = 16; int sweepers = 4; int threads = shards / (sweepers / 2); TimelockService stickyLockService = createStickyLockService(); createAndInitializeSweepersAndWaitForOneBackgroundIteration(sweepers, shards, threads, stickyLockService); ArgumentCaptor<LockRequest> captor = ArgumentCaptor.forClass(LockRequest.class); // minimum: as in the example above, but we have extra threads // threads + ... + threads * (shards / threads) + shards * (threads * sweepers - shards) verify(stickyLockService, atLeast(shards * (shards / threads + 1) / 2 + shards * (threads * sweepers - shards))) .lock(captor.capture()); // maximum: one would think that it is // shards + shards - 1 + ... + shards - (sweepers - 1) + shards * (threads * sweepers - shards) // but actually the logic is much more complicated since threads from the same sweeper can loop back and hit a // race condition with each other, so we go with the more conservative upper bound verify(stickyLockService, atMost(threads * sweepers * shards)).lock(any()); Set<String> requestedLockIds = captor.getAllValues().stream() .map(LockRequest::getLockDescriptors) .map(Iterables::getOnlyElement) .map(LockDescriptor::getLockIdAsString) .collect(Collectors.toSet()); Set<String> expectedLockIds = IntStream.range(0, shards).boxed() .map(ShardAndStrategy::conservative) .map(ShardAndStrategy::toText) .collect(Collectors.toSet()); assertThat(requestedLockIds).hasSameElementsAs(expectedLockIds); }
@Test public void multipleSweepersSweepDifferentShardsAndCallUnlockAfterwards() throws InterruptedException { int shards = 128; int sweepers = 8; int threads = shards / sweepers; TimelockService stickyLockService = createStickyLockService(); createAndInitializeSweepersAndWaitForOneBackgroundIteration(sweepers, shards, threads, stickyLockService); for (int i = 0; i < shards; i++) { assertProgressUpdatedToTimestamp(maxTsForFinePartition(tsPartitionFine(unreadableTs - 1)), i); verify(stickyLockService, times(1)).unlock(ImmutableSet.of(LockToken.of(new UUID(i, 0L)))); } // minimum: all threads on one host succeed, then on another, etc: // threads + threads * 2 + ... + threads * swepers verify(stickyLockService, atLeast(threads * sweepers * (sweepers - 1) / 2)) .lock(any(LockRequest.class)); // maximum: all but one succeed on each host, and only then those succeed: // shards + shards - 1 + ... + shards - (sweepers - 1) verify(stickyLockService, atMost(sweepers * shards - sweepers * (sweepers - 1) / 2)) .lock(any(LockRequest.class)); }