public SweepBatchConfig getAdjustedSweepConfig() { SweepBatchConfig sweepConfig = getRawSweepConfig(); double multiplier = batchSizeMultiplier; return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(adjust(sweepConfig.maxCellTsPairsToExamine(), multiplier)) .candidateBatchSize(adjust(sweepConfig.candidateBatchSize(), multiplier)) .deleteBatchSize(adjust(sweepConfig.deleteBatchSize(), multiplier)) .build(); }
void runOnceAndSaveResults(TableToSweep tableToSweep, SweepBatchConfig batchConfig) { TableReference tableRef = tableToSweep.getTableRef(); byte[] startRow = tableToSweep.getStartRow(); SweepResults results = runOneIteration(tableRef, startRow, batchConfig); processSweepResults(tableToSweep, results); }
SweepResults runOneIteration(TableReference tableRef, byte[] startRow, SweepBatchConfig batchConfig) { try { SweepResults results = sweepRunner.run(tableRef, batchConfig, startRow); logSweepPerformance(tableRef, startRow, results); return results; } catch (RuntimeException e) { // This error may be logged on some paths above, but I prefer to log defensively. logSweepError(tableRef, startRow, batchConfig, e); throw e; } }
SweepResults sweepTable(TableReference table) { SweepBatchConfig sweepConfig = ImmutableSweepBatchConfig.builder() .candidateBatchSize(AtlasDbConstants.DEFAULT_SWEEP_CANDIDATE_BATCH_HINT) .deleteBatchSize(AtlasDbConstants.DEFAULT_SWEEP_DELETE_BATCH_HINT) .maxCellTsPairsToExamine(AtlasDbConstants.DEFAULT_SWEEP_READ_LIMIT) .build(); return sweepTaskRunner.get().run(table, sweepConfig, PtBytes.EMPTY_BYTE_ARRAY); }
private SweepBatchConfig buildConfigWithOverrides( Optional<Integer> maxCellTsPairsToExamine, Optional<Integer> candidateBatchSize, Optional<Integer> deleteBatchSize) { ImmutableSweepBatchConfig.Builder batchConfigBuilder = ImmutableSweepBatchConfig.builder() .from(sweepBatchConfigSource.getAdjustedSweepConfig()); maxCellTsPairsToExamine.ifPresent(batchConfigBuilder::maxCellTsPairsToExamine); candidateBatchSize.ifPresent(batchConfigBuilder::candidateBatchSize); deleteBatchSize.ifPresent(batchConfigBuilder::deleteBatchSize); return batchConfigBuilder.build(); }
private static BatchOfCellsToSweep batch(List<CellToSweep> cells, int numCellTsPairsExamined, Cell lastExamined) { return ImmutableBatchOfCellsToSweep.builder() .cells(cells) .numCellTsPairsExamined(numCellTsPairsExamined) .lastCellExamined(lastExamined) .build(); }
private Optional<TableToSweep> getNextTableToSweep(Transaction tx, SweepPriorityOverrideConfig overrideConfig) { Optional<TableToSweep> nextTableToSweep = nextTableToSweepProvider.getNextTableToSweep( tx, specificTableSweeper.getSweepRunner().getConservativeSweepTimestamp(), overrideConfig); if (nextTableToSweep.isPresent()) { // Check if we're resuming this table after a previous sweep nextTableToSweep = augmentWithProgress(nextTableToSweep.get()); currentTable = nextTableToSweep; } return nextTableToSweep; }
private void waitUntilSpecificTableSweeperIsInitialized() throws InterruptedException { while (!specificTableSweeper.isInitialized()) { log.info("Sweep Priority Table and Sweep Progress Table are not initialized yet. If you have enabled " + "asynchronous initialization, these tables are being initialized asynchronously. Background " + "sweeper will start once the initialization is complete."); sleepFor(getBackoffTimeWhenSweepHasNotRun()); } }
@Override @Before public void setup() { super.setup(); sweeperService = new SweeperServiceImpl(specificTableSweeper, sweepBatchConfigSource); }
private void updatePreviousValues() { previousMultiplier = adjustableConfig.getBatchSizeMultiplier(); previousConfig = adjustableConfig.getAdjustedSweepConfig(); } }
private void updateBatchSize(SweepOutcome outcome) { if (outcome == SweepOutcome.SUCCESS) { sweepBatchConfigSource.increaseMultiplier(); } if (outcome == SweepOutcome.ERROR) { sweepBatchConfigSource.decreaseMultiplier(); } }
public void acquirePersistentLockWithRetry() { while (!tryAcquirePersistentLock()) { waitForRetry(); } }
protected void putTwoValuesInEachCell(List<Cell> cells) { createTable(SweepStrategy.CONSERVATIVE); int ts = 1; for (Cell cell : cells) { put(cell, "val1", ts); put(cell, "val2", ts + 5); ts += 10; } }
protected SpecificTableSweeper getSpecificTableSweeperService() { return new SpecificTableSweeper( SweeperTestSetup.mockTxManager(), kvs, sweepTaskRunner, priorityStore, progressStore, mock(BackgroundSweeperPerformanceLogger.class), sweepMetrics, () -> currentTimeMillis); }
private long setSweepTimestampAndGet(long timestamp) { immutableTs = timestamp; return Sweeper.CONSERVATIVE.getSweepTimestamp(timestampsSupplier); } }
public SweepResults dryRun(TableReference tableRef, SweepBatchConfig batchConfig, byte[] startRow) { return runInternal(tableRef, batchConfig, startRow, RunType.DRY); }
private SweepResults runOneBatchWithoutSavingResults( TableReference tableRef, byte[] startRow, SweepBatchConfig sweepBatchConfig) { return specificTableSweeper.runOneIteration( tableRef, startRow, sweepBatchConfig); }
private void configWithValues(int maxCellTsPairsToExamine, int candidateBatchSize, int deleteBatchSize) { adjustableConfig = AdjustableSweepBatchConfigSource.create( MetricsManagers.createForTests(), () -> ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(maxCellTsPairsToExamine) .candidateBatchSize(candidateBatchSize) .deleteBatchSize(deleteBatchSize) .build() ); updatePreviousValues(); }
@BeforeClass public static void initialiseConfig() { ImmutableSweepBatchConfig sweepBatchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(100) .candidateBatchSize(200) .maxCellTsPairsToExamine(1000) .build(); sweepBatchConfigSource = AdjustableSweepBatchConfigSource.create(metricsManager, () -> sweepBatchConfig); }
@Test public void canCreateConfig() { ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(1) .candidateBatchSize(1) .deleteBatchSize(1) .build(); }