private SweepBatchConfig buildConfigWithOverrides( Optional<Integer> maxCellTsPairsToExamine, Optional<Integer> candidateBatchSize, Optional<Integer> deleteBatchSize) { ImmutableSweepBatchConfig.Builder batchConfigBuilder = ImmutableSweepBatchConfig.builder() .from(sweepBatchConfigSource.getAdjustedSweepConfig()); maxCellTsPairsToExamine.ifPresent(batchConfigBuilder::maxCellTsPairsToExamine); candidateBatchSize.ifPresent(batchConfigBuilder::candidateBatchSize); deleteBatchSize.ifPresent(batchConfigBuilder::deleteBatchSize); return batchConfigBuilder.build(); }
@SuppressWarnings("unchecked") private Pair<List<List<Cell>>, SweepResults> runSweep(CellsSweeper cellsSweeper, SweepTaskRunner spiedSweepRunner, int maxCellTsPairsToExamine, int candidateBatchSize, int deleteBatchSize) { sweepTimestamp.set(Long.MAX_VALUE); List<List<Cell>> sweptCells = Lists.newArrayList(); doAnswer((invocationOnMock) -> { Object[] arguments = invocationOnMock.getArguments(); Collection<Cell> sentinelsToAdd = (Collection<Cell>) arguments[2]; sweptCells.add(new ArrayList(sentinelsToAdd)); return null; }).when(cellsSweeper).sweepCells(eq(TABLE_NAME), any(), any()); SweepResults sweepResults = spiedSweepRunner.run(TABLE_NAME, ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(maxCellTsPairsToExamine) .candidateBatchSize(candidateBatchSize) .deleteBatchSize(deleteBatchSize) .build(), PtBytes.EMPTY_BYTE_ARRAY); return new Pair(sweptCells, sweepResults); }
SweepResults sweepTable(TableReference table) { SweepBatchConfig sweepConfig = ImmutableSweepBatchConfig.builder() .candidateBatchSize(AtlasDbConstants.DEFAULT_SWEEP_CANDIDATE_BATCH_HINT) .deleteBatchSize(AtlasDbConstants.DEFAULT_SWEEP_DELETE_BATCH_HINT) .maxCellTsPairsToExamine(AtlasDbConstants.DEFAULT_SWEEP_READ_LIMIT) .build(); return sweepTaskRunner.get().run(table, sweepConfig, PtBytes.EMPTY_BYTE_ARRAY); }
private SweepResults partialSweep(long ts) { sweepTimestamp.set(ts); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(1) .candidateBatchSize(1) .maxCellTsPairsToExamine(1) .build(), PtBytes.EMPTY_BYTE_ARRAY); assertTrue(results.getNextStartRow().isPresent()); return results; } }
private static SweepBatchConfig getSweepBatchConfig(SweepConfig sweepConfig) { return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(sweepConfig.readLimit()) .candidateBatchSize(sweepConfig.candidateBatchHint() .orElse(AtlasDbConstants.DEFAULT_SWEEP_CANDIDATE_BATCH_HINT)) .deleteBatchSize(sweepConfig.deleteBatchHint()) .build(); }
private Object runMultiSweep(RegeneratingTable table) { SweepTaskRunner sweepTaskRunner = table.getSweepTaskRunner(); SweepResults sweepResults = null; byte[] nextStartRow = PtBytes.EMPTY_BYTE_ARRAY; for (int i = 0; i < BATCH_SIZE; i++) { SweepBatchConfig batchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(DELETED_COUNT) .candidateBatchSize(1) .maxCellTsPairsToExamine(RegeneratingTable.SWEEP_DUPLICATES) .build(); sweepResults = sweepTaskRunner.run(table.getTableRef(), batchConfig, nextStartRow); nextStartRow = sweepResults.getNextStartRow().get(); assertThat(sweepResults.getStaleValuesDeleted(), is((long) DELETED_COUNT)); } return sweepResults; }
private Object runSingleSweep(RegeneratingTable table, int uniqueCellsToSweep) { SweepTaskRunner sweepTaskRunner = table.getSweepTaskRunner(); SweepBatchConfig batchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(DELETED_COUNT * uniqueCellsToSweep) .candidateBatchSize(RegeneratingTable.SWEEP_DUPLICATES * uniqueCellsToSweep + 1) .maxCellTsPairsToExamine(RegeneratingTable.SWEEP_DUPLICATES * uniqueCellsToSweep) .build(); SweepResults sweepResults = sweepTaskRunner.run(table.getTableRef(), batchConfig, PtBytes.EMPTY_BYTE_ARRAY); assertThat(sweepResults.getStaleValuesDeleted(), is((long) DELETED_COUNT * uniqueCellsToSweep)); return sweepResults; }
private SweepBatchConfig getSweepBatchConfig() { if (batchSize != null || cellBatchSize != null) { printer.warn("Options 'batchSize' and 'cellBatchSize' have been deprecated in favor of 'deleteBatchHint', " + "'candidateBatchHint' and 'readLimit'. Please use the new options in the future."); } return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(chooseBestValue( readLimit, cellBatchSize, AtlasDbConstants.DEFAULT_SWEEP_READ_LIMIT)) .candidateBatchSize(chooseBestValue( candidateBatchHint, batchSize, AtlasDbConstants.DEFAULT_SWEEP_CANDIDATE_BATCH_HINT)) .deleteBatchSize(deleteBatchHint) .build(); }
SweepResults results = sweepRunner.run( tableReference, ImmutableSweepBatchConfig.builder() .deleteBatchSize(batchSize) .candidateBatchSize(batchSize)
@Test public void canCreateConfig() { ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(1) .candidateBatchSize(1) .deleteBatchSize(1) .build(); }
@Test(timeout = 50000) public void testSweeperFailsHalfwayThroughOnDeleteTable() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo2", "bang", 75); putIntoDefaultColumn("foo3", "baz", 100); putIntoDefaultColumn("foo4", "buzz", 125); byte[] nextStartRow = partialSweep(150).getNextStartRow().get(); kvs.dropTable(TABLE_NAME); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), nextStartRow); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); }
@Test(timeout = 50000) public void testSweepStrategyNothing() { createTable(SweepStrategy.NOTHING); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "bad", 75); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "buzz", 125); putUncommitted("foo", "foo", 150); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), PtBytes.EMPTY_BYTE_ARRAY); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); assertEquals(ImmutableSet.of(50L, 75L, 100L, 125L, 150L), getAllTsFromDefaultColumn("foo")); }
public SweepBatchConfig getAdjustedSweepConfig() { SweepBatchConfig sweepConfig = getRawSweepConfig(); double multiplier = batchSizeMultiplier; return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(adjust(sweepConfig.maxCellTsPairsToExamine(), multiplier)) .candidateBatchSize(adjust(sweepConfig.candidateBatchSize(), multiplier)) .deleteBatchSize(adjust(sweepConfig.deleteBatchSize(), multiplier)) .build(); }
@BeforeClass public static void initialiseConfig() { ImmutableSweepBatchConfig sweepBatchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(100) .candidateBatchSize(200) .maxCellTsPairsToExamine(1000) .build(); sweepBatchConfigSource = AdjustableSweepBatchConfigSource.create(metricsManager, () -> sweepBatchConfig); }
@Test public void canNotCreateConfigWithZeroCandidateBatchSize() { exception.expect(IllegalStateException.class); exception.expectMessage("Candidate batch size"); ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(1) .candidateBatchSize(0) .deleteBatchSize(1) .build(); }
@Test public void canNotCreateConfigWithZeroCellsToExamine() { exception.expect(IllegalStateException.class); exception.expectMessage("cells to examine"); ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(0) .candidateBatchSize(1) .deleteBatchSize(1) .build(); }
@Test public void canNotCreateConfigWithZeroDeleteBatchSize() { exception.expect(IllegalStateException.class); exception.expectMessage("Delete batch size"); ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(1) .candidateBatchSize(1) .deleteBatchSize(0) .build(); } }
private void configWithValues(int maxCellTsPairsToExamine, int candidateBatchSize, int deleteBatchSize) { adjustableConfig = AdjustableSweepBatchConfigSource.create( MetricsManagers.createForTests(), () -> ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(maxCellTsPairsToExamine) .candidateBatchSize(candidateBatchSize) .deleteBatchSize(deleteBatchSize) .build() ); updatePreviousValues(); }
private SweepBatchConfig buildConfigWithOverrides( Optional<Integer> maxCellTsPairsToExamine, Optional<Integer> candidateBatchSize, Optional<Integer> deleteBatchSize) { ImmutableSweepBatchConfig.Builder batchConfigBuilder = ImmutableSweepBatchConfig.builder() .from(sweepBatchConfigSource.getAdjustedSweepConfig()); maxCellTsPairsToExamine.ifPresent(batchConfigBuilder::maxCellTsPairsToExamine); candidateBatchSize.ifPresent(batchConfigBuilder::candidateBatchSize); deleteBatchSize.ifPresent(batchConfigBuilder::deleteBatchSize); return batchConfigBuilder.build(); }
public SweepBatchConfig getAdjustedSweepConfig() { SweepBatchConfig sweepConfig = getRawSweepConfig(); double multiplier = batchSizeMultiplier; return ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(adjust(sweepConfig.maxCellTsPairsToExamine(), multiplier)) .candidateBatchSize(adjust(sweepConfig.candidateBatchSize(), multiplier)) .deleteBatchSize(adjust(sweepConfig.deleteBatchSize(), multiplier)) .build(); }