private SweepBatchConfig buildConfigWithOverrides( Optional<Integer> maxCellTsPairsToExamine, Optional<Integer> candidateBatchSize, Optional<Integer> deleteBatchSize) { ImmutableSweepBatchConfig.Builder batchConfigBuilder = ImmutableSweepBatchConfig.builder() .from(sweepBatchConfigSource.getAdjustedSweepConfig()); maxCellTsPairsToExamine.ifPresent(batchConfigBuilder::maxCellTsPairsToExamine); candidateBatchSize.ifPresent(batchConfigBuilder::candidateBatchSize); deleteBatchSize.ifPresent(batchConfigBuilder::deleteBatchSize); return batchConfigBuilder.build(); }
@VisibleForTesting SweepOutcome runOnce() { Optional<TableToSweep> tableToSweep = getTableToSweep(); if (!tableToSweep.isPresent()) { // Don't change this log statement. It's parsed by test automation code. log.debug( "Skipping sweep because no table has enough new writes to be worth sweeping at the moment."); return SweepOutcome.NOTHING_TO_SWEEP; } SweepBatchConfig batchConfig = sweepBatchConfigSource.getAdjustedSweepConfig(); try { specificTableSweeper.runOnceAndSaveResults(tableToSweep.get(), batchConfig); return SweepOutcome.SUCCESS; } catch (InsufficientConsistencyException e) { log.info("Could not sweep because not all nodes of the database are online.", e); return SweepOutcome.NOT_ENOUGH_DB_NODES_ONLINE; } catch (RuntimeException e) { specificTableSweeper.updateSweepErrorMetric(); return determineCauseOfFailure(e, tableToSweep.get()); } }
private void updatePreviousValues() { previousMultiplier = adjustableConfig.getBatchSizeMultiplier(); previousConfig = adjustableConfig.getAdjustedSweepConfig(); } }
private void decreasesToOne(Function<SweepBatchConfig, Integer> getValue) { int newValue = getValue.apply(adjustableConfig.getAdjustedSweepConfig()); int previousValue = getValue.apply(previousConfig); assertThat(newValue, is(anyOf(equalTo(1), lessThan(previousValue)))); }
private void increasesBackUpToBaseConfig(Function<SweepBatchConfig, Integer> getValue) { assertThat(getValue.apply(adjustableConfig.getAdjustedSweepConfig()), is(anyOf( greaterThan(getValue.apply(previousConfig)), lessThanOrEqualTo(getValue.apply(adjustableConfig.getRawSweepConfig())) ))); }
public void decreaseMultiplier() { successiveIncreases.set(0); SweepBatchConfig lastBatchConfig = getAdjustedSweepConfig(); // Cut batch size in half, always sweep at least one row. reduceBatchSizeMultiplier(); log.info("Sweep failed unexpectedly with candidate batch size {}," + " delete batch size {}," + " and {} cell+timestamp pairs to examine." + " Attempting to continue with new batchSizeMultiplier {}", SafeArg.of("candidateBatchSize", lastBatchConfig.candidateBatchSize()), SafeArg.of("deleteBatchSize", lastBatchConfig.deleteBatchSize()), SafeArg.of("maxCellTsPairsToExamine", lastBatchConfig.maxCellTsPairsToExamine()), SafeArg.of("batchSizeMultiplier", batchSizeMultiplier)); }
private SweepBatchConfig buildConfigWithOverrides( Optional<Integer> maxCellTsPairsToExamine, Optional<Integer> candidateBatchSize, Optional<Integer> deleteBatchSize) { ImmutableSweepBatchConfig.Builder batchConfigBuilder = ImmutableSweepBatchConfig.builder() .from(sweepBatchConfigSource.getAdjustedSweepConfig()); maxCellTsPairsToExamine.ifPresent(batchConfigBuilder::maxCellTsPairsToExamine); candidateBatchSize.ifPresent(batchConfigBuilder::candidateBatchSize); deleteBatchSize.ifPresent(batchConfigBuilder::deleteBatchSize); return batchConfigBuilder.build(); }
@VisibleForTesting SweepOutcome runOnce() { Optional<TableToSweep> tableToSweep = getTableToSweep(); if (!tableToSweep.isPresent()) { // Don't change this log statement. It's parsed by test automation code. log.debug( "Skipping sweep because no table has enough new writes to be worth sweeping at the moment."); return SweepOutcome.NOTHING_TO_SWEEP; } SweepBatchConfig batchConfig = sweepBatchConfigSource.getAdjustedSweepConfig(); try { specificTableSweeper.runOnceAndSaveResults(tableToSweep.get(), batchConfig); return SweepOutcome.SUCCESS; } catch (InsufficientConsistencyException e) { log.info("Could not sweep because not all nodes of the database are online.", e); return SweepOutcome.NOT_ENOUGH_DB_NODES_ONLINE; } catch (RuntimeException e) { specificTableSweeper.updateSweepErrorMetric(); return determineCauseOfFailure(e, tableToSweep.get()); } }
public void decreaseMultiplier() { successiveIncreases.set(0); SweepBatchConfig lastBatchConfig = getAdjustedSweepConfig(); // Cut batch size in half, always sweep at least one row. reduceBatchSizeMultiplier(); log.info("Sweep failed unexpectedly with candidate batch size {}," + " delete batch size {}," + " and {} cell+timestamp pairs to examine." + " Attempting to continue with new batchSizeMultiplier {}", SafeArg.of("candidateBatchSize", lastBatchConfig.candidateBatchSize()), SafeArg.of("deleteBatchSize", lastBatchConfig.deleteBatchSize()), SafeArg.of("maxCellTsPairsToExamine", lastBatchConfig.maxCellTsPairsToExamine()), SafeArg.of("batchSizeMultiplier", batchSizeMultiplier)); }