private void updateBatchSize(SweepOutcome outcome) { if (outcome == SweepOutcome.SUCCESS) { sweepBatchConfigSource.increaseMultiplier(); } if (outcome == SweepOutcome.ERROR) { sweepBatchConfigSource.decreaseMultiplier(); } }
private void updatePreviousValues() { previousMultiplier = adjustableConfig.getBatchSizeMultiplier(); previousConfig = adjustableConfig.getAdjustedSweepConfig(); } }
private void increasesBackUpToBaseConfig(Function<SweepBatchConfig, Integer> getValue) { assertThat(getValue.apply(adjustableConfig.getAdjustedSweepConfig()), is(anyOf( greaterThan(getValue.apply(previousConfig)), lessThanOrEqualTo(getValue.apply(adjustableConfig.getRawSweepConfig())) ))); }
public static AdjustableSweepBatchConfigSource create( MetricsManager metricsManager, Supplier<SweepBatchConfig> rawSweepBatchConfig) { AdjustableSweepBatchConfigSource configSource = new AdjustableSweepBatchConfigSource(rawSweepBatchConfig); metricsManager.registerMetric(AdjustableSweepBatchConfigSource.class, "batchSizeMultiplier", () -> getBatchSizeMultiplier()); return configSource; }
sweepMetrics); BackgroundSweeperPerformanceLogger sweepPerfLogger = new NoOpBackgroundSweeperPerformanceLogger(); AdjustableSweepBatchConfigSource sweepBatchConfigSource = AdjustableSweepBatchConfigSource.create( metricsManager, () -> getSweepBatchConfig(runtimeConfigSupplier.get().sweep()));
private void batchSizeMultiplierIncreases() { assertThat(adjustableConfig.getBatchSizeMultiplier(), is(greaterThanOrEqualTo(previousMultiplier))); }
public SweepBatchConfig getAdjustedSweepConfig() { SweepBatchConfig sweepConfig = getRawSweepConfig(); double multiplier = batchSizeMultiplier; return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(adjust(sweepConfig.maxCellTsPairsToExamine(), multiplier)) .candidateBatchSize(adjust(sweepConfig.candidateBatchSize(), multiplier)) .deleteBatchSize(adjust(sweepConfig.deleteBatchSize(), multiplier)) .build(); }
public void decreaseMultiplier() { successiveIncreases.set(0); SweepBatchConfig lastBatchConfig = getAdjustedSweepConfig(); // Cut batch size in half, always sweep at least one row. reduceBatchSizeMultiplier(); log.info("Sweep failed unexpectedly with candidate batch size {}," + " delete batch size {}," + " and {} cell+timestamp pairs to examine." + " Attempting to continue with new batchSizeMultiplier {}", SafeArg.of("candidateBatchSize", lastBatchConfig.candidateBatchSize()), SafeArg.of("deleteBatchSize", lastBatchConfig.deleteBatchSize()), SafeArg.of("maxCellTsPairsToExamine", lastBatchConfig.maxCellTsPairsToExamine()), SafeArg.of("batchSizeMultiplier", batchSizeMultiplier)); }
@Test public void batchSizeMultiplierDecreasesOnFailure() { // Given configWithValues(1000, 1000, 1000); // When adjustableConfig.decreaseMultiplier(); // Then assertThat(adjustableConfig.getBatchSizeMultiplier(), is(lessThan(previousMultiplier))); }
private SweepBatchConfig buildConfigWithOverrides( Optional<Integer> maxCellTsPairsToExamine, Optional<Integer> candidateBatchSize, Optional<Integer> deleteBatchSize) { ImmutableSweepBatchConfig.Builder batchConfigBuilder = ImmutableSweepBatchConfig.builder() .from(sweepBatchConfigSource.getAdjustedSweepConfig()); maxCellTsPairsToExamine.ifPresent(batchConfigBuilder::maxCellTsPairsToExamine); candidateBatchSize.ifPresent(batchConfigBuilder::candidateBatchSize); deleteBatchSize.ifPresent(batchConfigBuilder::deleteBatchSize); return batchConfigBuilder.build(); }
private void reduceBatchSizeMultiplier() { SweepBatchConfig config = getRawSweepConfig(); double smallestSensibleBatchSizeMultiplier = 1.0 / NumberUtils.max( config.maxCellTsPairsToExamine(), config.candidateBatchSize(), config.deleteBatchSize()); if (batchSizeMultiplier == smallestSensibleBatchSizeMultiplier) { return; } double newBatchSizeMultiplier = batchSizeMultiplier / 2; if (newBatchSizeMultiplier < smallestSensibleBatchSizeMultiplier) { log.info("batchSizeMultiplier reached the smallest sensible value for the current sweep config ({}), " + "will not reduce further.", SafeArg.of("batchSizeMultiplier", smallestSensibleBatchSizeMultiplier)); batchSizeMultiplier = smallestSensibleBatchSizeMultiplier; } else { batchSizeMultiplier = newBatchSizeMultiplier; } } }
@BeforeClass public static void initialiseConfig() { ImmutableSweepBatchConfig sweepBatchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(100) .candidateBatchSize(200) .maxCellTsPairsToExamine(1000) .build(); sweepBatchConfigSource = AdjustableSweepBatchConfigSource.create(metricsManager, () -> sweepBatchConfig); }
private void batchSizeMultiplierDoesNotExceedOne() { assertThat(adjustableConfig.getBatchSizeMultiplier(), is(lessThanOrEqualTo(1.0))); }
public SweepBatchConfig getAdjustedSweepConfig() { SweepBatchConfig sweepConfig = getRawSweepConfig(); double multiplier = batchSizeMultiplier; return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(adjust(sweepConfig.maxCellTsPairsToExamine(), multiplier)) .candidateBatchSize(adjust(sweepConfig.candidateBatchSize(), multiplier)) .deleteBatchSize(adjust(sweepConfig.deleteBatchSize(), multiplier)) .build(); }
public static AdjustableSweepBatchConfigSource create( MetricsManager metricsManager, Supplier<SweepBatchConfig> rawSweepBatchConfig) { AdjustableSweepBatchConfigSource configSource = new AdjustableSweepBatchConfigSource(rawSweepBatchConfig); metricsManager.registerMetric(AdjustableSweepBatchConfigSource.class, "batchSizeMultiplier", () -> getBatchSizeMultiplier()); return configSource; }
public void decreaseMultiplier() { successiveIncreases.set(0); SweepBatchConfig lastBatchConfig = getAdjustedSweepConfig(); // Cut batch size in half, always sweep at least one row. reduceBatchSizeMultiplier(); log.info("Sweep failed unexpectedly with candidate batch size {}," + " delete batch size {}," + " and {} cell+timestamp pairs to examine." + " Attempting to continue with new batchSizeMultiplier {}", SafeArg.of("candidateBatchSize", lastBatchConfig.candidateBatchSize()), SafeArg.of("deleteBatchSize", lastBatchConfig.deleteBatchSize()), SafeArg.of("maxCellTsPairsToExamine", lastBatchConfig.maxCellTsPairsToExamine()), SafeArg.of("batchSizeMultiplier", batchSizeMultiplier)); }
@VisibleForTesting SweepOutcome runOnce() { Optional<TableToSweep> tableToSweep = getTableToSweep(); if (!tableToSweep.isPresent()) { // Don't change this log statement. It's parsed by test automation code. log.debug( "Skipping sweep because no table has enough new writes to be worth sweeping at the moment."); return SweepOutcome.NOTHING_TO_SWEEP; } SweepBatchConfig batchConfig = sweepBatchConfigSource.getAdjustedSweepConfig(); try { specificTableSweeper.runOnceAndSaveResults(tableToSweep.get(), batchConfig); return SweepOutcome.SUCCESS; } catch (InsufficientConsistencyException e) { log.info("Could not sweep because not all nodes of the database are online.", e); return SweepOutcome.NOT_ENOUGH_DB_NODES_ONLINE; } catch (RuntimeException e) { specificTableSweeper.updateSweepErrorMetric(); return determineCauseOfFailure(e, tableToSweep.get()); } }
private void reduceBatchSizeMultiplier() { SweepBatchConfig config = getRawSweepConfig(); double smallestSensibleBatchSizeMultiplier = 1.0 / NumberUtils.max( config.maxCellTsPairsToExamine(), config.candidateBatchSize(), config.deleteBatchSize()); if (batchSizeMultiplier == smallestSensibleBatchSizeMultiplier) { return; } double newBatchSizeMultiplier = batchSizeMultiplier / 2; if (newBatchSizeMultiplier < smallestSensibleBatchSizeMultiplier) { log.info("batchSizeMultiplier reached the smallest sensible value for the current sweep config ({}), " + "will not reduce further.", SafeArg.of("batchSizeMultiplier", smallestSensibleBatchSizeMultiplier)); batchSizeMultiplier = smallestSensibleBatchSizeMultiplier; } else { batchSizeMultiplier = newBatchSizeMultiplier; } } }
private void configWithValues(int maxCellTsPairsToExamine, int candidateBatchSize, int deleteBatchSize) { adjustableConfig = AdjustableSweepBatchConfigSource.create( MetricsManagers.createForTests(), () -> ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(maxCellTsPairsToExamine) .candidateBatchSize(candidateBatchSize) .deleteBatchSize(deleteBatchSize) .build() ); updatePreviousValues(); }
private void batchSizeMultiplierDecreases() { assertThat(adjustableConfig.getBatchSizeMultiplier(), is(lessThanOrEqualTo(previousMultiplier))); }