@Value.Check default void check() { Preconditions.checkState(maxCellTsPairsToExamine() > 0, "Number of cells to examine must be greater than zero"); Preconditions.checkState(candidateBatchSize() > 0, "Candidate batch size must be greater than zero"); Preconditions.checkState(deleteBatchSize() > 0, "Delete batch size must be greater than zero"); } }
public void decreaseMultiplier() { successiveIncreases.set(0); SweepBatchConfig lastBatchConfig = getAdjustedSweepConfig(); // Cut batch size in half, always sweep at least one row. reduceBatchSizeMultiplier(); log.info("Sweep failed unexpectedly with candidate batch size {}," + " delete batch size {}," + " and {} cell+timestamp pairs to examine." + " Attempting to continue with new batchSizeMultiplier {}", SafeArg.of("candidateBatchSize", lastBatchConfig.candidateBatchSize()), SafeArg.of("deleteBatchSize", lastBatchConfig.deleteBatchSize()), SafeArg.of("maxCellTsPairsToExamine", lastBatchConfig.maxCellTsPairsToExamine()), SafeArg.of("batchSizeMultiplier", batchSizeMultiplier)); }
private void reduceBatchSizeMultiplier() { SweepBatchConfig config = getRawSweepConfig(); double smallestSensibleBatchSizeMultiplier = 1.0 / NumberUtils.max( config.maxCellTsPairsToExamine(), config.candidateBatchSize(), config.deleteBatchSize()); if (batchSizeMultiplier == smallestSensibleBatchSizeMultiplier) { return; } double newBatchSizeMultiplier = batchSizeMultiplier / 2; if (newBatchSizeMultiplier < smallestSensibleBatchSizeMultiplier) { log.info("batchSizeMultiplier reached the smallest sensible value for the current sweep config ({}), " + "will not reduce further.", SafeArg.of("batchSizeMultiplier", smallestSensibleBatchSizeMultiplier)); batchSizeMultiplier = smallestSensibleBatchSizeMultiplier; } else { batchSizeMultiplier = newBatchSizeMultiplier; } } }
private void logSweepError(TableReference tableRef, byte[] startRow, SweepBatchConfig config, RuntimeException exception) { log.info("Failed to sweep table {}" + " at row {}" + " with candidate batch size {}," + " delete batch size {}," + " and {} cell+timestamp pairs to examine.", LoggingArgs.tableRef("tableRef", tableRef), UnsafeArg.of("startRow", startRowToHex(startRow)), SafeArg.of("candidateBatchSize", config.candidateBatchSize()), SafeArg.of("deleteBatchSize", config.deleteBatchSize()), SafeArg.of("maxCellTsPairsToExamine", config.maxCellTsPairsToExamine()), exception); }
private SweepResults runSweep(Optional<Boolean> fullSweep, TableReference tableRef, byte[] decodedStartRow, SweepBatchConfig config) { if (!fullSweep.isPresent()) { log.warn("fullSweep parameter was not specified, defaulting to true"); } if (fullSweep.orElse(true)) { log.info("Running sweep of full table {}, " + "with maxCellTsPairsToExamine: {}, candidateBatchSize: {}, deleteBatchSize: {}, " + "starting from row {}", LoggingArgs.tableRef(tableRef), SafeArg.of("maxCellTsPairsToExamine", config.maxCellTsPairsToExamine()), SafeArg.of("candidateBatchSize", config.candidateBatchSize()), SafeArg.of("deleteBatchSize", config.deleteBatchSize()), UnsafeArg.of("decodedStartRow", decodedStartRow)); return runFullSweepWithoutSavingResults(tableRef, decodedStartRow, config); } else { log.info("Running sweep of a single batch on table {}, " + "with maxCellTsPairsToExamine: {}, candidateBatchSize: {}, deleteBatchSize: {}, " + "starting from row {}", LoggingArgs.tableRef(tableRef), SafeArg.of("maxCellTsPairsToExamine", config.maxCellTsPairsToExamine()), SafeArg.of("candidateBatchSize", config.candidateBatchSize()), SafeArg.of("deleteBatchSize", config.deleteBatchSize()), UnsafeArg.of("decodedStartRow", decodedStartRow)); return runOneBatchWithoutSavingResults(tableRef, decodedStartRow, config); } }
public SweepBatchConfig getAdjustedSweepConfig() { SweepBatchConfig sweepConfig = getRawSweepConfig(); double multiplier = batchSizeMultiplier; return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(adjust(sweepConfig.maxCellTsPairsToExamine(), multiplier)) .candidateBatchSize(adjust(sweepConfig.candidateBatchSize(), multiplier)) .deleteBatchSize(adjust(sweepConfig.deleteBatchSize(), multiplier)) .build(); }
CandidateCellForSweepingRequest request = ImmutableCandidateCellForSweepingRequest.builder() .startRowInclusive(startRow) .batchSizeHint(batchConfig.candidateBatchSize()) .maxTimestampExclusive(sweepTs) .shouldCheckIfLatestValueIsEmpty(sweeper.shouldSweepLastCommitted())
@Value.Check default void check() { Preconditions.checkState(maxCellTsPairsToExamine() > 0, "Number of cells to examine must be greater than zero"); Preconditions.checkState(candidateBatchSize() > 0, "Candidate batch size must be greater than zero"); Preconditions.checkState(deleteBatchSize() > 0, "Delete batch size must be greater than zero"); } }
private void reduceBatchSizeMultiplier() { SweepBatchConfig config = getRawSweepConfig(); double smallestSensibleBatchSizeMultiplier = 1.0 / NumberUtils.max( config.maxCellTsPairsToExamine(), config.candidateBatchSize(), config.deleteBatchSize()); if (batchSizeMultiplier == smallestSensibleBatchSizeMultiplier) { return; } double newBatchSizeMultiplier = batchSizeMultiplier / 2; if (newBatchSizeMultiplier < smallestSensibleBatchSizeMultiplier) { log.info("batchSizeMultiplier reached the smallest sensible value for the current sweep config ({}), " + "will not reduce further.", SafeArg.of("batchSizeMultiplier", smallestSensibleBatchSizeMultiplier)); batchSizeMultiplier = smallestSensibleBatchSizeMultiplier; } else { batchSizeMultiplier = newBatchSizeMultiplier; } } }
public void decreaseMultiplier() { successiveIncreases.set(0); SweepBatchConfig lastBatchConfig = getAdjustedSweepConfig(); // Cut batch size in half, always sweep at least one row. reduceBatchSizeMultiplier(); log.info("Sweep failed unexpectedly with candidate batch size {}," + " delete batch size {}," + " and {} cell+timestamp pairs to examine." + " Attempting to continue with new batchSizeMultiplier {}", SafeArg.of("candidateBatchSize", lastBatchConfig.candidateBatchSize()), SafeArg.of("deleteBatchSize", lastBatchConfig.deleteBatchSize()), SafeArg.of("maxCellTsPairsToExamine", lastBatchConfig.maxCellTsPairsToExamine()), SafeArg.of("batchSizeMultiplier", batchSizeMultiplier)); }
private SweepResults runSweep(Optional<Boolean> fullSweep, TableReference tableRef, byte[] decodedStartRow, SweepBatchConfig config) { if (!fullSweep.isPresent()) { log.warn("fullSweep parameter was not specified, defaulting to true"); } if (fullSweep.orElse(true)) { log.info("Running sweep of full table {}, " + "with maxCellTsPairsToExamine: {}, candidateBatchSize: {}, deleteBatchSize: {}, " + "starting from row {}", LoggingArgs.tableRef(tableRef), SafeArg.of("maxCellTsPairsToExamine", config.maxCellTsPairsToExamine()), SafeArg.of("candidateBatchSize", config.candidateBatchSize()), SafeArg.of("deleteBatchSize", config.deleteBatchSize()), UnsafeArg.of("decodedStartRow", decodedStartRow)); return runFullSweepWithoutSavingResults(tableRef, decodedStartRow, config); } else { log.info("Running sweep of a single batch on table {}, " + "with maxCellTsPairsToExamine: {}, candidateBatchSize: {}, deleteBatchSize: {}, " + "starting from row {}", LoggingArgs.tableRef(tableRef), SafeArg.of("maxCellTsPairsToExamine", config.maxCellTsPairsToExamine()), SafeArg.of("candidateBatchSize", config.candidateBatchSize()), SafeArg.of("deleteBatchSize", config.deleteBatchSize()), UnsafeArg.of("decodedStartRow", decodedStartRow)); return runOneBatchWithoutSavingResults(tableRef, decodedStartRow, config); } }
private void logSweepError(TableReference tableRef, byte[] startRow, SweepBatchConfig config, RuntimeException exception) { log.info("Failed to sweep table {}" + " at row {}" + " with candidate batch size {}," + " delete batch size {}," + " and {} cell+timestamp pairs to examine.", LoggingArgs.tableRef("tableRef", tableRef), UnsafeArg.of("startRow", startRowToHex(startRow)), SafeArg.of("candidateBatchSize", config.candidateBatchSize()), SafeArg.of("deleteBatchSize", config.deleteBatchSize()), SafeArg.of("maxCellTsPairsToExamine", config.maxCellTsPairsToExamine()), exception); }
public SweepBatchConfig getAdjustedSweepConfig() { SweepBatchConfig sweepConfig = getRawSweepConfig(); double multiplier = batchSizeMultiplier; return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(adjust(sweepConfig.maxCellTsPairsToExamine(), multiplier)) .candidateBatchSize(adjust(sweepConfig.candidateBatchSize(), multiplier)) .deleteBatchSize(adjust(sweepConfig.deleteBatchSize(), multiplier)) .build(); }
CandidateCellForSweepingRequest request = ImmutableCandidateCellForSweepingRequest.builder() .startRowInclusive(startRow) .batchSizeHint(batchConfig.candidateBatchSize()) .maxTimestampExclusive(sweepTs) .shouldCheckIfLatestValueIsEmpty(sweeper.shouldSweepLastCommitted())