SweepResults runOneIteration(TableReference tableRef, byte[] startRow, SweepBatchConfig batchConfig) { try { SweepResults results = sweepRunner.run(tableRef, batchConfig, startRow); logSweepPerformance(tableRef, startRow, results); return results; } catch (RuntimeException e) { // This error may be logged on some paths above, but I prefer to log defensively. logSweepError(tableRef, startRow, batchConfig, e); throw e; } }
protected void setupTaskRunner(TableReference tableRef, SweepResults results) { doReturn(results).when(sweepTaskRunner).run(eq(tableRef), any(), any()); }
@SuppressWarnings("unchecked") private Pair<List<List<Cell>>, SweepResults> runSweep(CellsSweeper cellsSweeper, SweepTaskRunner spiedSweepRunner, int maxCellTsPairsToExamine, int candidateBatchSize, int deleteBatchSize) { sweepTimestamp.set(Long.MAX_VALUE); List<List<Cell>> sweptCells = Lists.newArrayList(); doAnswer((invocationOnMock) -> { Object[] arguments = invocationOnMock.getArguments(); Collection<Cell> sentinelsToAdd = (Collection<Cell>) arguments[2]; sweptCells.add(new ArrayList(sentinelsToAdd)); return null; }).when(cellsSweeper).sweepCells(eq(TABLE_NAME), any(), any()); SweepResults sweepResults = spiedSweepRunner.run(TABLE_NAME, ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(maxCellTsPairsToExamine) .candidateBatchSize(candidateBatchSize) .deleteBatchSize(deleteBatchSize) .build(), PtBytes.EMPTY_BYTE_ARRAY); return new Pair(sweptCells, sweepResults); }
SweepResults sweepTable(TableReference table) { SweepBatchConfig sweepConfig = ImmutableSweepBatchConfig.builder() .candidateBatchSize(AtlasDbConstants.DEFAULT_SWEEP_CANDIDATE_BATCH_HINT) .deleteBatchSize(AtlasDbConstants.DEFAULT_SWEEP_DELETE_BATCH_HINT) .maxCellTsPairsToExamine(AtlasDbConstants.DEFAULT_SWEEP_READ_LIMIT) .build(); return sweepTaskRunner.get().run(table, sweepConfig, PtBytes.EMPTY_BYTE_ARRAY); }
@Test public void runsOneIterationIfRequested() { setupTaskRunner(RESULTS_MORE_TO_SWEEP); sweeperService.sweepTable(TABLE_REF.getQualifiedName(), Optional.empty(), Optional.of(false), Optional.empty(), Optional.empty(), Optional.empty()); verify(sweepTaskRunner, times(1)).run(any(), any(), any()); verifyNoMoreInteractions(sweepTaskRunner); }
@Test public void sweepsEntireTableByDefault() { List<byte[]> startRows = ImmutableList.of( PtBytes.EMPTY_BYTE_ARRAY, new byte[] {0x10}, new byte[] {0x50}); for (int i = 0; i < startRows.size(); i++) { byte[] currentRow = startRows.get(i); Optional<byte[]> nextRow = (i + 1) == startRows.size() ? Optional.empty() : Optional.of(startRows.get(i + 1)); SweepResults results = SweepResults.createEmptySweepResult(nextRow); when(sweepTaskRunner.run(any(), any(), eq(currentRow))).thenReturn(results); } sweeperService.sweepTableFully(TABLE_REF.getQualifiedName()); startRows.forEach(row -> verify(sweepTaskRunner).run(any(), any(), eq(row))); verifyNoMoreInteractions(sweepTaskRunner); }
private SweepResults partialSweep(long ts) { sweepTimestamp.set(ts); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(1) .candidateBatchSize(1) .maxCellTsPairsToExamine(1) .build(), PtBytes.EMPTY_BYTE_ARRAY); assertTrue(results.getNextStartRow().isPresent()); return results; } }
private Object runMultiSweep(RegeneratingTable table) { SweepTaskRunner sweepTaskRunner = table.getSweepTaskRunner(); SweepResults sweepResults = null; byte[] nextStartRow = PtBytes.EMPTY_BYTE_ARRAY; for (int i = 0; i < BATCH_SIZE; i++) { SweepBatchConfig batchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(DELETED_COUNT) .candidateBatchSize(1) .maxCellTsPairsToExamine(RegeneratingTable.SWEEP_DUPLICATES) .build(); sweepResults = sweepTaskRunner.run(table.getTableRef(), batchConfig, nextStartRow); nextStartRow = sweepResults.getNextStartRow().get(); assertThat(sweepResults.getStaleValuesDeleted(), is((long) DELETED_COUNT)); } return sweepResults; }
private Object runSingleSweep(RegeneratingTable table, int uniqueCellsToSweep) { SweepTaskRunner sweepTaskRunner = table.getSweepTaskRunner(); SweepBatchConfig batchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(DELETED_COUNT * uniqueCellsToSweep) .candidateBatchSize(RegeneratingTable.SWEEP_DUPLICATES * uniqueCellsToSweep + 1) .maxCellTsPairsToExamine(RegeneratingTable.SWEEP_DUPLICATES * uniqueCellsToSweep) .build(); SweepResults sweepResults = sweepTaskRunner.run(table.getTableRef(), batchConfig, PtBytes.EMPTY_BYTE_ARRAY); assertThat(sweepResults.getStaleValuesDeleted(), is((long) DELETED_COUNT * uniqueCellsToSweep)); return sweepResults; }
long totalCellsExamined = 0; for (int run = 0; run < 100; ++run) { SweepResults results = sweepRunner.run( tableReference, ImmutableSweepBatchConfig.builder()
@Test(timeout = 50000) public void testSweeperFailsHalfwayThroughOnDeleteTable() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo2", "bang", 75); putIntoDefaultColumn("foo3", "baz", 100); putIntoDefaultColumn("foo4", "buzz", 125); byte[] nextStartRow = partialSweep(150).getNextStartRow().get(); kvs.dropTable(TABLE_NAME); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), nextStartRow); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); }
@Test(timeout = 50000) public void testSweepStrategyNothing() { createTable(SweepStrategy.NOTHING); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "bad", 75); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "buzz", 125); putUncommitted("foo", "foo", 150); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), PtBytes.EMPTY_BYTE_ARRAY); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); assertEquals(ImmutableSet.of(50L, 75L, 100L, 125L, 150L), getAllTsFromDefaultColumn("foo")); }
SweepResults newResults = dryRun ? sweepRunner.dryRun(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()) : sweepRunner.run(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get());
SweepResults runOneIteration(TableReference tableRef, byte[] startRow, SweepBatchConfig batchConfig) { try { SweepResults results = sweepRunner.run(tableRef, batchConfig, startRow); logSweepPerformance(tableRef, startRow, results); return results; } catch (RuntimeException e) { // This error may be logged on some paths above, but I prefer to log defensively. logSweepError(tableRef, startRow, batchConfig, e); throw e; } }
@SuppressWarnings("unchecked") private Pair<List<List<Cell>>, SweepResults> runSweep(CellsSweeper cellsSweeper, SweepTaskRunner spiedSweepRunner, int maxCellTsPairsToExamine, int candidateBatchSize, int deleteBatchSize) { sweepTimestamp.set(Long.MAX_VALUE); List<List<Cell>> sweptCells = Lists.newArrayList(); doAnswer((invocationOnMock) -> { Object[] arguments = invocationOnMock.getArguments(); Collection<Cell> sentinelsToAdd = (Collection<Cell>) arguments[2]; sweptCells.add(new ArrayList(sentinelsToAdd)); return null; }).when(cellsSweeper).sweepCells(eq(TABLE_NAME), any(), any()); SweepResults sweepResults = spiedSweepRunner.run(TABLE_NAME, ImmutableSweepBatchConfig.builder() .maxCellTsPairsToExamine(maxCellTsPairsToExamine) .candidateBatchSize(candidateBatchSize) .deleteBatchSize(deleteBatchSize) .build(), PtBytes.EMPTY_BYTE_ARRAY); return new Pair(sweptCells, sweepResults); }
private SweepResults partialSweep(long ts) { sweepTimestamp.set(ts); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(1) .candidateBatchSize(1) .maxCellTsPairsToExamine(1) .build(), PtBytes.EMPTY_BYTE_ARRAY); assertTrue(results.getNextStartRow().isPresent()); return results; } }
long totalCellsExamined = 0; for (int run = 0; run < 100; ++run) { SweepResults results = sweepRunner.run( tableReference, ImmutableSweepBatchConfig.builder()
@Test(timeout = 50000) public void testSweepStrategyNothing() { createTable(SweepStrategy.NOTHING); putIntoDefaultColumn("foo", "bar", 50); putUncommitted("foo", "bad", 75); putIntoDefaultColumn("foo", "baz", 100); putIntoDefaultColumn("foo", "buzz", 125); putUncommitted("foo", "foo", 150); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), PtBytes.EMPTY_BYTE_ARRAY); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); assertEquals(ImmutableSet.of(50L, 75L, 100L, 125L, 150L), getAllTsFromDefaultColumn("foo")); }
@Test(timeout = 50000) public void testSweeperFailsHalfwayThroughOnDeleteTable() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo2", "bang", 75); putIntoDefaultColumn("foo3", "baz", 100); putIntoDefaultColumn("foo4", "buzz", 125); byte[] nextStartRow = partialSweep(150).getNextStartRow().get(); kvs.dropTable(TABLE_NAME); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), nextStartRow); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); }
SweepResults newResults = dryRun ? sweepRunner.dryRun(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()) : sweepRunner.run(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get());