@Test public void canUpgradeNumberOfShardsIfPersistedDefaultValue() { byte[] defaultValue = ShardProgress.createColumnValue(AtlasDbConstants.DEFAULT_SWEEP_QUEUE_SHARDS); CheckAndSetRequest request = progress.createNewCellRequest(ShardProgress.SHARD_COUNT_SAS, defaultValue); kvs.checkAndSet(request); progress.updateNumberOfShards(128); assertThat(progress.getNumberOfShards()).isEqualTo(128); }
@Test public void updatingTimestampsDoesNotAffectShardsAndViceVersa() { assertThat(progress.getNumberOfShards()).isEqualTo(AtlasDbConstants.DEFAULT_SWEEP_QUEUE_SHARDS); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(INITIAL_TIMESTAMP); assertThat(progress.getLastSweptTimestamp(THOROUGH_TEN)).isEqualTo(INITIAL_TIMESTAMP); progress.updateNumberOfShards(64); progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 32L); progress.updateLastSweptTimestamp(THOROUGH_TEN, 128L); assertThat(progress.getNumberOfShards()).isEqualTo(64); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(32L); assertThat(progress.getLastSweptTimestamp(THOROUGH_TEN)).isEqualTo(128L); }
private CheckAndSetRequest createSingleCellRequest(ShardAndStrategy shardAndStrategy, long oldVal, byte[] colValNew) { byte[] colValOld = createColumnValue(oldVal); return CheckAndSetRequest.singleCell(TABLE_REF, cellForShard(shardAndStrategy), colValOld, colValNew); } }
/** * Updates the persisted last swept timestamp for the given shard and strategy to timestamp if it is greater than * the currently persisted last swept timestamp. * * @param shardAndStrategy shard and strategy to update for * @param timestamp timestamp to update to * @return the latest known persisted sweep timestamp for the shard and strategy */ public long updateLastSweptTimestamp(ShardAndStrategy shardAndStrategy, long timestamp) { return increaseValueFromToAtLeast(shardAndStrategy, getLastSweptTimestamp(shardAndStrategy), timestamp); }
private CheckAndSetRequest createRequest(ShardAndStrategy shardAndStrategy, long oldVal, byte[] colValNew) { if (isDefaultValue(shardAndStrategy, oldVal)) { return maybeGet(shardAndStrategy) .map(persistedValue -> createSingleCellRequest(shardAndStrategy, persistedValue, colValNew)) .orElse(createNewCellRequest(shardAndStrategy, colValNew)); } else { return createSingleCellRequest(shardAndStrategy, oldVal, colValNew); } }
/** * Updates the persisted number of shards to newNumber, if newNumber is greater than the currently persisted number * of shards. * * @param newNumber the desired new number of shards * @return the latest known persisted number of shards, which may be greater than newNumber */ public int updateNumberOfShards(int newNumber) { Preconditions.checkArgument(newNumber <= AtlasDbConstants.MAX_SWEEP_QUEUE_SHARDS); return (int) increaseValueFromToAtLeast(SHARD_COUNT_SAS, getNumberOfShards(), newNumber); }
@Test public void attemptingToDecreaseNumberOfShardsIsNoop() { progress.updateNumberOfShards(64); progress.updateNumberOfShards(32); assertThat(progress.getNumberOfShards()).isEqualTo(64); }
@Test public void updatingTimestampForOneShardDoesNotAffectOthers() { assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(INITIAL_TIMESTAMP); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TWENTY)).isEqualTo(INITIAL_TIMESTAMP); progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 1024L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TWENTY)).isEqualTo(INITIAL_TIMESTAMP); progress.updateLastSweptTimestamp(CONSERVATIVE_TWENTY, 512L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TWENTY)).isEqualTo(512L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(1024L); }
private void progressTo(ShardAndStrategy shardStrategy, long newProgress) { if (newProgress < 0) { log.warn("Wasn't able to progress targeted sweep for {} since last swept timestamp {} is negative.", SafeArg.of("shardStrategy", shardStrategy.toText()), SafeArg.of("timestamp", newProgress)); return; } progress.updateLastSweptTimestamp(shardStrategy, newProgress); log.debug("Progressed last swept timestamp for {} to {}.", SafeArg.of("shardStrategy", shardStrategy.toText()), SafeArg.of("timestamp", newProgress)); }
@Test public void canReadInitialSweptTimestamp() { assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(INITIAL_TIMESTAMP); }
private long increaseValueFromToAtLeast(ShardAndStrategy shardAndStrategy, long oldVal, long newVal) { byte[] colValNew = createColumnValue(newVal); long currentValue = oldVal; while (currentValue < newVal) { CheckAndSetRequest casRequest = createRequest(shardAndStrategy, currentValue, colValNew); try { kvs.checkAndSet(casRequest); return newVal; } catch (CheckAndSetException e) { log.info("Failed to check and set from expected old value {} to new value {}. Retrying if the old " + "value changed under us.", SafeArg.of("old value", currentValue), SafeArg.of("new value", newVal)); currentValue = rethrowIfUnchanged(shardAndStrategy, currentValue, e); } } return currentValue; }
@Test public void repeatedlyFailingCasThrows() { KeyValueService mockKvs = mock(KeyValueService.class); when(mockKvs.get(any(), anyMap())) .thenReturn(ImmutableMap.of()) .thenReturn(ImmutableMap.of(DUMMY, createValue(5L))) .thenReturn(ImmutableMap.of(DUMMY, createValue(10L))) .thenReturn(ImmutableMap.of(DUMMY, createValue(10L))); doThrow(new CheckAndSetException("sadness")).when(mockKvs).checkAndSet(any()); ShardProgress instrumentedProgress = new ShardProgress(mockKvs); assertThatThrownBy(() -> instrumentedProgress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 12L)) .isInstanceOf(CheckAndSetException.class); }
@Before public void setup() { kvs = new InMemoryKeyValueService(true); progress = new ShardProgress(kvs); }
/** * Creates a supplier such that the first call to {@link Supplier#get()} on it will take the maximum of the runtime * configuration and the persisted number of shards, and persist and memoize the result. Subsequent calls will * return the cached value until refreshTimeMillis has passed, at which point the next call will again perform the * check and set. * * @param runtimeConfig live reloadable runtime configuration for the number of shards * @param progress progress table persisting the number of shards * @param refreshTimeMillis timeout for caching the number of shards * @return supplier calculating and persisting the number of shards to use */ public static Supplier<Integer> createProgressUpdatingSupplier(Supplier<Integer> runtimeConfig, ShardProgress progress, long refreshTimeMillis) { return Suppliers.memoizeWithExpiration( () -> progress.updateNumberOfShards(runtimeConfig.get()), refreshTimeMillis, TimeUnit.MILLISECONDS); }
@Test public void canReadInitialNumberOfShards() { assertThat(progress.getNumberOfShards()).isEqualTo(AtlasDbConstants.DEFAULT_SWEEP_QUEUE_SHARDS); }
private Map<Cell, Value> getEntry(ShardAndStrategy shardAndStrategy) { return kvs.get(TABLE_REF, ImmutableMap.of(cellForShard(shardAndStrategy), SweepQueueUtils.READ_TS)); }
private CheckAndSetRequest createRequest(ShardAndStrategy shardAndStrategy, long oldVal, byte[] colValNew) { if (isDefaultValue(shardAndStrategy, oldVal)) { return maybeGet(shardAndStrategy) .map(persistedValue -> createSingleCellRequest(shardAndStrategy, persistedValue, colValNew)) .orElse(createNewCellRequest(shardAndStrategy, colValNew)); } else { return createSingleCellRequest(shardAndStrategy, oldVal, colValNew); } }
@Test public void cannotUpdateNumberOfShardsToZero() { progress.updateNumberOfShards(0); assertThat(progress.getNumberOfShards()).isEqualTo(AtlasDbConstants.DEFAULT_SWEEP_QUEUE_SHARDS); }
@Test public void attemptingToDecreaseSweptTimestampIsNoop() { progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 1024L); progress.updateLastSweptTimestamp(CONSERVATIVE_TEN, 512L); assertThat(progress.getLastSweptTimestamp(CONSERVATIVE_TEN)).isEqualTo(1024L); }
@Test public void canReadNextWhenOtherShardsAndStrategiesProgressToEndOfPartitionForThorough() { progress.updateLastSweptTimestamp(thorough(shardCons), maxTsForFinePartition(TS2_FINE_PARTITION)); progress.updateLastSweptTimestamp(conservative(shardThor), maxTsForFinePartition(TS2_FINE_PARTITION)); progress.updateLastSweptTimestamp(conservative(shardCons), maxTsForFinePartition(TS2_FINE_PARTITION)); assertThat(readThorough(shardThor)).contains(TS2_FINE_PARTITION); }