private Optional<TableToSweep> augmentWithProgress(TableToSweep nextTableWithoutProgress) { Optional<SweepProgress> sweepProgress = specificTableSweeper.getSweepProgressStore().loadProgress( nextTableWithoutProgress.getTableRef()); if (sweepProgress.isPresent()) { TableToSweep nextTableWithProgress = TableToSweep.continueSweeping(nextTableWithoutProgress.getTableRef(), nextTableWithoutProgress.getSweepLock(), sweepProgress.get()); return Optional.of(nextTableWithProgress); } return Optional.of(nextTableWithoutProgress); }
private SweepOutcome determineCauseOfFailure(Exception originalException, TableToSweep tableToSweep) { try { Set<TableReference> tables = specificTableSweeper.getKvs().getAllTableNames(); if (!tables.contains(tableToSweep.getTableRef())) { clearSweepProgress(tableToSweep.getTableRef()); log.info("The table being swept by the background sweeper was dropped, moving on..."); tableToSweep.getSweepLock().close(); return SweepOutcome.TABLE_DROPPED_WHILE_SWEEPING; } log.info("The background sweep job failed unexpectedly; will retry with a lower batch size...", originalException); return SweepOutcome.ERROR; } catch (RuntimeException newE) { log.warn("Sweep failed", originalException); log.warn("Failed to check whether the table being swept was dropped. Retrying...", newE); return SweepOutcome.ERROR; } }
private Optional<TableToSweep> logDecision( Optional<TableToSweep> chosenTable, Map<TableReference, Double> scores, SweepPriorityOverrideConfig sweepPriorityOverrideConfig) { if (!log.isDebugEnabled()) { return chosenTable; } String safeTableNamesToScore = scores.entrySet().stream() .sorted(Comparator.comparingDouble(Map.Entry::getValue)) .map(entry -> LoggingArgs.safeTableOrPlaceholder(entry.getKey()) + "->" + entry.getValue()) .collect(Collectors.joining(", ", "[", "]")); String chosenTableString = chosenTable.isPresent() ? LoggingArgs.safeTableOrPlaceholder(chosenTable.get().getTableRef()).toString() : "no table"; log.debug("Chose {} from scores: {}, unsafeScores: {}, overrides: {}", SafeArg.of("chosenTable", chosenTableString), SafeArg.of("scores", safeTableNamesToScore), UnsafeArg.of("unsafeScores", scores), UnsafeArg.of("overrides", sweepPriorityOverrideConfig)); return chosenTable; } }
void runOnceAndSaveResults(TableToSweep tableToSweep, SweepBatchConfig batchConfig) { TableReference tableRef = tableToSweep.getTableRef(); byte[] startRow = tableToSweep.getStartRow(); SweepResults results = runOneIteration(tableRef, startRow, batchConfig); processSweepResults(tableToSweep, results); }
private void thenTableChosenIs(TableReference table) { Optional<TableToSweep> tableToSweep = Iterables.getOnlyElement(tablesToSweep); Assert.assertTrue("expected to have chosen a table!", tableToSweep.isPresent()); Assert.assertThat(tableToSweep.get().getTableRef(), is(table)); }
private Optional<TableToSweep> getTableToSweep() { return specificTableSweeper.getTxManager().runTaskWithRetry( tx -> { Optional<SweepProgress> progress = currentTable.flatMap( tableToSweep -> specificTableSweeper.getSweepProgressStore().loadProgress( tableToSweep.getTableRef())); SweepPriorityOverrideConfig overrideConfig = sweepPriorityOverrideConfig.get(); if (progress.map( realProgress -> shouldContinueSweepingCurrentTable(realProgress, overrideConfig)) .orElse(false)) { try { // If we're here, currentTable exists and we're going to sweep it again this iteration updateProgressAndRefreshLock(progress.get()); return currentTable; } catch (InterruptedException ex) { log.info("Sweep lost the lock for table {}", LoggingArgs.tableRef(progress.get().tableRef())); closeTableLockIfHeld(); currentTable = Optional.empty(); // We'll fall through and choose a new table } } log.info("Sweep is choosing a new table to sweep."); closeTableLockIfHeld(); return getNextTableToSweep(tx, overrideConfig); }); }
private void processFinishedSweep(TableToSweep tableToSweep, SweepResults cumulativeResults) { saveFinalSweepResults(tableToSweep, cumulativeResults); log.info("Finished sweeping table {}. Examined {} cell+timestamp pairs, deleted {} stale values. Time taken " + "sweeping: {} ms, time elapsed since sweep first started on this table: {} ms.", LoggingArgs.tableRef("tableRef", tableToSweep.getTableRef()), SafeArg.of("cellTs pairs examined", cumulativeResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", cumulativeResults.getStaleValuesDeleted()), SafeArg.of("time sweeping table", cumulativeResults.getTimeInMillis()), SafeArg.of("time elapsed", cumulativeResults.getTimeElapsedSinceStartedSweeping())); tableToSweep.getSweepLock().close(); sweepProgressStore.clearProgress(tableToSweep.getTableRef()); }
private void saveIntermediateSweepResults(TableToSweep tableToSweep, SweepResults results) { Preconditions.checkArgument(results.getNextStartRow().isPresent(), "Next start row should be present when saving intermediate results!"); txManager.runTaskWithRetry((TxTask) tx -> { if (!tableToSweep.hasPreviousProgress()) { // This is the first set of results being written for this table. sweepPriorityStore.update( tx, tableToSweep.getTableRef(), ImmutableUpdateSweepPriority.builder().newWriteCount(0L).build()); } SweepProgress newProgress = ImmutableSweepProgress.builder() .tableRef(tableToSweep.getTableRef()) .staleValuesDeleted(results.getStaleValuesDeleted()) .cellTsPairsExamined(results.getCellTsPairsExamined()) //noinspection OptionalGetWithoutIsPresent // covered by precondition above .startRow(results.getNextStartRow().get()) .startColumn(PtBytes.toBytes("unused")) .minimumSweptTimestamp(results.getMinSweptTimestamp()) .timeInMillis(results.getTimeInMillis()) .startTimeInMillis(results.getTimeSweepStarted()) .build(); sweepProgressStore.saveProgress(newProgress); return null; }); }
@Test public void calculatorReturnsManyTablesWithHighestPriority_thenProviderReturnsOneOfThose() { givenPriority(table("table1"), 0.0); givenPriority(table("table2"), 30.0); givenPriority(table("table3"), 30.0); givenPriority(table("table4"), 30.0); givenPriority(table("table5"), 10.0); whenGettingNextTableToSweep(); Optional<TableToSweep> tableToSweep = Iterables.getOnlyElement(tablesToSweep); Assert.assertTrue(tableToSweep.isPresent()); Assert.assertThat(tableToSweep.get().getTableRef(), anyOf(is(table("table2")), is(table("table3")), is(table("table4")))); }
private void saveFinalSweepResults(TableToSweep tableToSweep, SweepResults finalSweepResults) { txManager.runTaskWithRetry((TxTask) tx -> { ImmutableUpdateSweepPriority.Builder update = ImmutableUpdateSweepPriority.builder() .newStaleValuesDeleted(finalSweepResults.getStaleValuesDeleted()) .newCellTsPairsExamined(finalSweepResults.getCellTsPairsExamined()) .newLastSweepTimeMillis(wallClock.getTimeMillis()) .newMinimumSweptTimestamp(finalSweepResults.getMinSweptTimestamp()); if (!tableToSweep.hasPreviousProgress()) { // This is the first (and only) set of results being written for this table. update.newWriteCount(0L); } sweepPriorityStore.update(tx, tableToSweep.getTableRef(), update.build()); return null; }); }
private Optional<TableToSweep> augmentWithProgress(TableToSweep nextTableWithoutProgress) { Optional<SweepProgress> sweepProgress = specificTableSweeper.getSweepProgressStore().loadProgress( nextTableWithoutProgress.getTableRef()); if (sweepProgress.isPresent()) { TableToSweep nextTableWithProgress = TableToSweep.continueSweeping(nextTableWithoutProgress.getTableRef(), nextTableWithoutProgress.getSweepLock(), sweepProgress.get()); return Optional.of(nextTableWithProgress); } return Optional.of(nextTableWithoutProgress); }
private SweepOutcome determineCauseOfFailure(Exception originalException, TableToSweep tableToSweep) { try { Set<TableReference> tables = specificTableSweeper.getKvs().getAllTableNames(); if (!tables.contains(tableToSweep.getTableRef())) { clearSweepProgress(tableToSweep.getTableRef()); log.info("The table being swept by the background sweeper was dropped, moving on..."); tableToSweep.getSweepLock().close(); return SweepOutcome.TABLE_DROPPED_WHILE_SWEEPING; } log.info("The background sweep job failed unexpectedly; will retry with a lower batch size...", originalException); return SweepOutcome.ERROR; } catch (RuntimeException newE) { log.warn("Sweep failed", originalException); log.warn("Failed to check whether the table being swept was dropped. Retrying...", newE); return SweepOutcome.ERROR; } }
private Optional<TableToSweep> logDecision( Optional<TableToSweep> chosenTable, Map<TableReference, Double> scores, SweepPriorityOverrideConfig sweepPriorityOverrideConfig) { if (!log.isDebugEnabled()) { return chosenTable; } String safeTableNamesToScore = scores.entrySet().stream() .sorted(Comparator.comparingDouble(Map.Entry::getValue)) .map(entry -> LoggingArgs.safeTableOrPlaceholder(entry.getKey()) + "->" + entry.getValue()) .collect(Collectors.joining(", ", "[", "]")); String chosenTableString = chosenTable.isPresent() ? LoggingArgs.safeTableOrPlaceholder(chosenTable.get().getTableRef()).toString() : "no table"; log.debug("Chose {} from scores: {}, unsafeScores: {}, overrides: {}", SafeArg.of("chosenTable", chosenTableString), SafeArg.of("scores", safeTableNamesToScore), UnsafeArg.of("unsafeScores", scores), UnsafeArg.of("overrides", sweepPriorityOverrideConfig)); return chosenTable; } }
void runOnceAndSaveResults(TableToSweep tableToSweep, SweepBatchConfig batchConfig) { TableReference tableRef = tableToSweep.getTableRef(); byte[] startRow = tableToSweep.getStartRow(); SweepResults results = runOneIteration(tableRef, startRow, batchConfig); processSweepResults(tableToSweep, results); }
private Optional<TableToSweep> getTableToSweep() { return specificTableSweeper.getTxManager().runTaskWithRetry( tx -> { Optional<SweepProgress> progress = currentTable.flatMap( tableToSweep -> specificTableSweeper.getSweepProgressStore().loadProgress( tableToSweep.getTableRef())); SweepPriorityOverrideConfig overrideConfig = sweepPriorityOverrideConfig.get(); if (progress.map( realProgress -> shouldContinueSweepingCurrentTable(realProgress, overrideConfig)) .orElse(false)) { try { // If we're here, currentTable exists and we're going to sweep it again this iteration updateProgressAndRefreshLock(progress.get()); return currentTable; } catch (InterruptedException ex) { log.info("Sweep lost the lock for table {}", LoggingArgs.tableRef(progress.get().tableRef())); closeTableLockIfHeld(); currentTable = Optional.empty(); // We'll fall through and choose a new table } } log.info("Sweep is choosing a new table to sweep."); closeTableLockIfHeld(); return getNextTableToSweep(tx, overrideConfig); }); }
private void processFinishedSweep(TableToSweep tableToSweep, SweepResults cumulativeResults) { saveFinalSweepResults(tableToSweep, cumulativeResults); log.info("Finished sweeping table {}. Examined {} cell+timestamp pairs, deleted {} stale values. Time taken " + "sweeping: {} ms, time elapsed since sweep first started on this table: {} ms.", LoggingArgs.tableRef("tableRef", tableToSweep.getTableRef()), SafeArg.of("cellTs pairs examined", cumulativeResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", cumulativeResults.getStaleValuesDeleted()), SafeArg.of("time sweeping table", cumulativeResults.getTimeInMillis()), SafeArg.of("time elapsed", cumulativeResults.getTimeElapsedSinceStartedSweeping())); tableToSweep.getSweepLock().close(); sweepProgressStore.clearProgress(tableToSweep.getTableRef()); }
private void saveIntermediateSweepResults(TableToSweep tableToSweep, SweepResults results) { Preconditions.checkArgument(results.getNextStartRow().isPresent(), "Next start row should be present when saving intermediate results!"); txManager.runTaskWithRetry((TxTask) tx -> { if (!tableToSweep.hasPreviousProgress()) { // This is the first set of results being written for this table. sweepPriorityStore.update( tx, tableToSweep.getTableRef(), ImmutableUpdateSweepPriority.builder().newWriteCount(0L).build()); } SweepProgress newProgress = ImmutableSweepProgress.builder() .tableRef(tableToSweep.getTableRef()) .staleValuesDeleted(results.getStaleValuesDeleted()) .cellTsPairsExamined(results.getCellTsPairsExamined()) //noinspection OptionalGetWithoutIsPresent // covered by precondition above .startRow(results.getNextStartRow().get()) .startColumn(PtBytes.toBytes("unused")) .minimumSweptTimestamp(results.getMinSweptTimestamp()) .timeInMillis(results.getTimeInMillis()) .startTimeInMillis(results.getTimeSweepStarted()) .build(); sweepProgressStore.saveProgress(newProgress); return null; }); }
private void saveFinalSweepResults(TableToSweep tableToSweep, SweepResults finalSweepResults) { txManager.runTaskWithRetry((TxTask) tx -> { ImmutableUpdateSweepPriority.Builder update = ImmutableUpdateSweepPriority.builder() .newStaleValuesDeleted(finalSweepResults.getStaleValuesDeleted()) .newCellTsPairsExamined(finalSweepResults.getCellTsPairsExamined()) .newLastSweepTimeMillis(wallClock.getTimeMillis()) .newMinimumSweptTimestamp(finalSweepResults.getMinSweptTimestamp()); if (!tableToSweep.hasPreviousProgress()) { // This is the first (and only) set of results being written for this table. update.newWriteCount(0L); } sweepPriorityStore.update(tx, tableToSweep.getTableRef(), update.build()); return null; }); }