@Test(timeout = 50000) public void testSweepingAlreadySweptTable() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("row", "val", 10); putIntoDefaultColumn("row", "val", 20); Optional<SweepResults> optResults = completeSweep(30); optResults.ifPresent(results -> { assertEquals(1, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2); }); optResults = completeSweep(40); optResults.ifPresent(results -> { assertEquals(0, results.getStaleValuesDeleted()); assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(1); }); }
SweepResults accumulatedResults = SweepResults.createEmptySweepResult(Optional.of(entry.getValue())); while (accumulatedResults.getNextStartRow().isPresent()) { SweepResults newResults = dryRun ? sweepRunner.dryRun(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()) : sweepRunner.run(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()); accumulatedResults = accumulatedResults.accumulateWith(newResults); printer.info( "{} Swept from {} to {} in table {} in {} ms, examined {} cell values," + " {} ms. Total time sweeping this table: {} ms.", SafeArg.of("isDryRun", dryRun ? "[DRY RUN]" : ""), UnsafeArg.of("startRow", encodeStartRow(accumulatedResults.getNextStartRow())), UnsafeArg.of("exclusiveEndRow", encodeEndRow(newResults.getNextStartRow())), LoggingArgs.tableRef(tableToSweep), SafeArg.of("time taken millis", newResults.getTimeInMillis()), SafeArg.of("cellTs pairs examined", newResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", newResults.getStaleValuesDeleted()), SafeArg.of("time elapsed", accumulatedResults.getTimeElapsedSinceStartedSweeping()), SafeArg.of("time sweeping", accumulatedResults.getTimeInMillis())); tableToSweep.getQualifiedName()); priorityTable.putWriteCount(row1, 0L); priorityTable.putCellsExamined(row1, finalAccumulatedResults.getCellTsPairsExamined()); priorityTable.putCellsDeleted(row1, finalAccumulatedResults.getStaleValuesDeleted()); priorityTable.putLastSweepTime(row1, System.currentTimeMillis()); return null;
/** * Returns a new {@link SweepResults} representing cumulative results from this instance and {@code other}. * The operation is commutative. */ public SweepResults accumulateWith(SweepResults other) { return SweepResults.builder() .nextStartRow(maxRowOptional(getNextStartRow(), other.getNextStartRow())) .cellTsPairsExamined(getCellTsPairsExamined() + other.getCellTsPairsExamined()) .staleValuesDeleted(getStaleValuesDeleted() + other.getStaleValuesDeleted()) .minSweptTimestamp(Math.min(getMinSweptTimestamp(), other.getMinSweptTimestamp())) .timeInMillis(getTimeInMillis() + other.getTimeInMillis()) .timeSweepStarted(Math.min(getTimeSweepStarted(), other.getTimeSweepStarted())) .build(); }
private SweepResults runFullSweepWithoutSavingResults( TableReference tableRef, byte[] startRow, SweepBatchConfig sweepBatchConfig) { SweepResults cumulativeResults = SweepResults.createEmptySweepResult(Optional.of(startRow)); while (cumulativeResults.getNextStartRow().isPresent()) { SweepResults results = runOneBatchWithoutSavingResults( tableRef, cumulativeResults.getNextStartRow().get(), sweepBatchConfig); specificTableSweeper.updateTimeMetricsOneIteration(results.getTimeInMillis(), results.getTimeElapsedSinceStartedSweeping()); cumulativeResults = cumulativeResults.accumulateWith(results); } return cumulativeResults; }
private void processSweepResults(TableToSweep tableToSweep, SweepResults currentIteration) { updateTimeMetricsOneIteration(currentIteration.getTimeInMillis(), currentIteration.getTimeElapsedSinceStartedSweeping()); SweepResults cumulativeResults = getCumulativeSweepResults(tableToSweep, currentIteration); if (currentIteration.getNextStartRow().isPresent()) { saveIntermediateSweepResults(tableToSweep, cumulativeResults); } else { processFinishedSweep(tableToSweep, cumulativeResults); } }
private void saveIntermediateSweepResults(TableToSweep tableToSweep, SweepResults results) { Preconditions.checkArgument(results.getNextStartRow().isPresent(), "Next start row should be present when saving intermediate results!"); txManager.runTaskWithRetry((TxTask) tx -> { if (!tableToSweep.hasPreviousProgress()) { // This is the first set of results being written for this table. sweepPriorityStore.update( tx, tableToSweep.getTableRef(), ImmutableUpdateSweepPriority.builder().newWriteCount(0L).build()); } SweepProgress newProgress = ImmutableSweepProgress.builder() .tableRef(tableToSweep.getTableRef()) .staleValuesDeleted(results.getStaleValuesDeleted()) .cellTsPairsExamined(results.getCellTsPairsExamined()) //noinspection OptionalGetWithoutIsPresent // covered by precondition above .startRow(results.getNextStartRow().get()) .startColumn(PtBytes.toBytes("unused")) .minimumSweptTimestamp(results.getMinSweptTimestamp()) .timeInMillis(results.getTimeInMillis()) .startTimeInMillis(results.getTimeSweepStarted()) .build(); sweepProgressStore.saveProgress(newProgress); return null; }); }
.build(), startRow); assertEquals(ts, results.getMinSweptTimestamp()); assertArrayEquals(startRow, results.getPreviousStartRow().orElse(null)); totalStaleValuesDeleted += results.getStaleValuesDeleted(); totalCellsExamined += results.getCellTsPairsExamined(); if (!results.getNextStartRow().isPresent()) { return Optional.of(SweepResults.builder() .staleValuesDeleted(totalStaleValuesDeleted) .cellTsPairsExamined(totalCellsExamined) .build()); startRow = results.getNextStartRow().get();
private void logSweepPerformance(TableReference tableRef, byte[] startRow, SweepResults results) { log.info("Analyzed {} cell+timestamp pairs" + " from table {}" + " starting at row {}" + " and deleted {} stale values" + " in {} ms" + " up to timestamp {}.", SafeArg.of("cellTs pairs examined", results.getCellTsPairsExamined()), LoggingArgs.tableRef("tableRef", tableRef), UnsafeArg.of("startRow", startRowToHex(startRow)), SafeArg.of("cellTs pairs deleted", results.getStaleValuesDeleted()), SafeArg.of("time taken", results.getTimeInMillis()), SafeArg.of("last swept timestamp", results.getMinSweptTimestamp())); SweepPerformanceResults performanceResults = SweepPerformanceResults.builder() .sweepResults(results) .tableName(tableRef.getQualifiedName()) .elapsedMillis(results.getTimeInMillis()) .build(); sweepPerfLogger.logSweepResults(performanceResults); }
private void processFinishedSweep(TableToSweep tableToSweep, SweepResults cumulativeResults) { saveFinalSweepResults(tableToSweep, cumulativeResults); log.info("Finished sweeping table {}. Examined {} cell+timestamp pairs, deleted {} stale values. Time taken " + "sweeping: {} ms, time elapsed since sweep first started on this table: {} ms.", LoggingArgs.tableRef("tableRef", tableToSweep.getTableRef()), SafeArg.of("cellTs pairs examined", cumulativeResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", cumulativeResults.getStaleValuesDeleted()), SafeArg.of("time sweeping table", cumulativeResults.getTimeInMillis()), SafeArg.of("time elapsed", cumulativeResults.getTimeElapsedSinceStartedSweeping())); tableToSweep.getSweepLock().close(); sweepProgressStore.clearProgress(tableToSweep.getTableRef()); }
static SweepTableResponse from(SweepResults results) { return ImmutableSweepTableResponse.builder() .numCellTsPairsExamined(results.getCellTsPairsExamined()) .staleValuesDeleted(results.getStaleValuesDeleted()) .nextStartRow(results.getNextStartRow().map(PtBytes::encodeHexString)) .build(); }
private void saveFinalSweepResults(TableToSweep tableToSweep, SweepResults finalSweepResults) { txManager.runTaskWithRetry((TxTask) tx -> { ImmutableUpdateSweepPriority.Builder update = ImmutableUpdateSweepPriority.builder() .newStaleValuesDeleted(finalSweepResults.getStaleValuesDeleted()) .newCellTsPairsExamined(finalSweepResults.getCellTsPairsExamined()) .newLastSweepTimeMillis(wallClock.getTimeMillis()) .newMinimumSweptTimestamp(finalSweepResults.getMinSweptTimestamp()); if (!tableToSweep.hasPreviousProgress()) { // This is the first (and only) set of results being written for this table. update.newWriteCount(0L); } sweepPriorityStore.update(tx, tableToSweep.getTableRef(), update.build()); return null; }); }
@Test public void equalsIgnoresTimeSweepStarted() { SweepResults emptySweepResult = SweepResults.createEmptySweepResult(Optional.empty()); SweepResults laterEmptySweepResult = SweepResults.builder().from(emptySweepResult) .timeSweepStarted(emptySweepResult.getTimeSweepStarted() + 1) .build(); Assert.assertThat(emptySweepResult, Matchers.equalTo(laterEmptySweepResult)); } }
public static SweepResults createEmptySweepResult(Optional<byte[]> startRow) { return builder() .cellTsPairsExamined(0) .staleValuesDeleted(0) .minSweptTimestamp(Long.MAX_VALUE) .nextStartRow(startRow) .timeInMillis(0) .timeSweepStarted(System.currentTimeMillis()) .build(); }
@Test public void testMetricsRecordedAfterIncompleteRunForOneIterationOnly() { setNoProgress(); setNextTableToSweep(TABLE_REF); SweepResults intermediateResults = SweepResults.builder() .staleValuesDeleted(2) .cellTsPairsExamined(10) .minSweptTimestamp(12345L) .nextStartRow(Optional.of(new byte[] {1, 2, 3})) .timeInMillis(10L) .timeSweepStarted(20L) .build(); setupTaskRunner(intermediateResults); backgroundSweeper.runOnce(); ArgumentCaptor<Long> sweepTime = ArgumentCaptor.forClass(Long.class); ArgumentCaptor<Long> totalTimeElapsed = ArgumentCaptor.forClass(Long.class); Mockito.verify(sweepMetrics).updateSweepTime( sweepTime.capture(), totalTimeElapsed.capture()); Assertions.assertThat(intermediateResults.getTimeInMillis()).isEqualTo(sweepTime.getValue()); Assertions.assertThat(intermediateResults.getTimeElapsedSinceStartedSweeping()) .isCloseTo(totalTimeElapsed.getValue(), Percentage.withPercentage(5d)); }
private Object runMultiSweep(RegeneratingTable table) { SweepTaskRunner sweepTaskRunner = table.getSweepTaskRunner(); SweepResults sweepResults = null; byte[] nextStartRow = PtBytes.EMPTY_BYTE_ARRAY; for (int i = 0; i < BATCH_SIZE; i++) { SweepBatchConfig batchConfig = ImmutableSweepBatchConfig.builder() .deleteBatchSize(DELETED_COUNT) .candidateBatchSize(1) .maxCellTsPairsToExamine(RegeneratingTable.SWEEP_DUPLICATES) .build(); sweepResults = sweepTaskRunner.run(table.getTableRef(), batchConfig, nextStartRow); nextStartRow = sweepResults.getNextStartRow().get(); assertThat(sweepResults.getStaleValuesDeleted(), is((long) DELETED_COUNT)); } return sweepResults; }
@Test(timeout = 50000) public void testSweeperFailsHalfwayThroughOnDeleteTable() { createTable(SweepStrategy.CONSERVATIVE); putIntoDefaultColumn("foo", "bar", 50); putIntoDefaultColumn("foo2", "bang", 75); putIntoDefaultColumn("foo3", "baz", 100); putIntoDefaultColumn("foo4", "buzz", 125); byte[] nextStartRow = partialSweep(150).getNextStartRow().get(); kvs.dropTable(TABLE_NAME); SweepResults results = sweepRunner.run( TABLE_NAME, ImmutableSweepBatchConfig.builder() .deleteBatchSize(DEFAULT_BATCH_SIZE) .candidateBatchSize(DEFAULT_BATCH_SIZE) .maxCellTsPairsToExamine(DEFAULT_BATCH_SIZE) .build(), nextStartRow); assertEquals(SweepResults.createEmptySweepResult(Optional.empty()), results); }
public static SweepResults createEmptySweepResultWithMoreToSweep() { return createEmptySweepResult(Optional.of(PtBytes.EMPTY_BYTE_ARRAY)); }
@Test public void shouldReturnValuesForMultipleColumnsWhenSweeping() { createTable(TableMetadataPersistence.SweepStrategy.CONSERVATIVE); for (int ts = 10; ts <= 150; ts += 10) { put("row", "col1", "value", ts); put("row", "col2", "value", ts + 5); } SweepResults results = completeSweep(350).get(); Assert.assertEquals(28, results.getStaleValuesDeleted()); }
private void verifyExpectedArgument(long sweepTime, long totalTimeElapsed) { assertThat(RESULTS_NO_MORE_TO_SWEEP.getTimeInMillis()).isEqualTo(sweepTime); assertThat(totalTimeElapsed).isBetween( RESULTS_NO_MORE_TO_SWEEP.getTimeElapsedSinceStartedSweeping() - 1000L, RESULTS_NO_MORE_TO_SWEEP.getTimeElapsedSinceStartedSweeping()); }
public byte[] getStartRow() { return previousResults.getNextStartRow().orElse(PtBytes.EMPTY_BYTE_ARRAY); }