private long endOfFinePartitionForTs(long timestamp) { return SweepQueueUtils.maxTsForFinePartition(tsPartitionFine(timestamp)); }
public static long maxTsForFinePartition(long finePartition) { return minTsForFinePartition(finePartition) + TS_FINE_GRANULARITY - 1; }
@Test public void fineToCoarseIsCorrect() { long prime = 674506111L; Assertions.assertThat(SweepQueueUtils.partitionFineToCoarse(SweepQueueUtils.tsPartitionFine(prime))) .isEqualTo(SweepQueueUtils.tsPartitionCoarse(prime)); } }
/** * Returns fine partition that should have unprocessed entries in the Sweepable Cells table. * * @param shardStrategy desired shard and strategy * @param lastSweptTs exclusive minimum timestamp to check for * @param sweepTs exclusive maximum timestamp to check for * @return Optional containing the fine partition, or Optional.empty() if there are no more candidates before * sweepTs */ Optional<Long> nextSweepableTimestampPartition(ShardAndStrategy shardStrategy, long lastSweptTs, long sweepTs) { long minFineInclusive = SweepQueueUtils.tsPartitionFine(lastSweptTs + 1); long maxFineInclusive = SweepQueueUtils.tsPartitionFine(sweepTs - 1); return nextSweepablePartition(shardStrategy, minFineInclusive, maxFineInclusive); }
private long lastGuaranteedSwept(long partitionFine, long maxTsExclusive) { return Math.min(SweepQueueUtils.maxTsForFinePartition(partitionFine), maxTsExclusive - 1); }
@Test public void noNextTimestampWhenSweepTimestampInSamePartitionAndLower() { immutableTs = minTsForFinePartition(TS_FINE_PARTITION); assertThat(tsPartitionFine(getSweepTsCons())).isEqualTo(TS_FINE_PARTITION); assertThat(getSweepTsCons()).isLessThan(TS); assertThat(readConservative(shardCons)).isEmpty(); }
private void cleanSweepableTimestamps(ShardAndStrategy shardStrategy, long oldProgress, long newProgress) { if (firstIterationOfSweep(oldProgress)) { return; } long lastSweptPartitionPreviously = SweepQueueUtils.tsPartitionCoarse(oldProgress); long minimumSweepPartitionNextIteration = SweepQueueUtils.tsPartitionCoarse(newProgress + 1); if (minimumSweepPartitionNextIteration > lastSweptPartitionPreviously) { sweepableTimestamps.deleteRow(shardStrategy, lastSweptPartitionPreviously); log.info("Deleted persisted sweep queue information in table {} for partition {}.", LoggingArgs.tableRef(TargetedSweepTableFactory.of() .getSweepableTimestampsTable(null).getTableRef()), SafeArg.of("partition", lastSweptPartitionPreviously)); } }
private Optional<Long> nextSweepablePartition(ShardAndStrategy shardAndStrategy, long minFineInclusive, long maxFineInclusive) { ColumnRangeSelection range = getColRangeSelection(minFineInclusive, maxFineInclusive + 1); long current = SweepQueueUtils.partitionFineToCoarse(minFineInclusive); long maxCoarseInclusive = SweepQueueUtils.partitionFineToCoarse(maxFineInclusive); while (current <= maxCoarseInclusive) { Optional<Long> candidateFine = getCandidatesInCoarsePartition(shardAndStrategy, current, range); if (candidateFine.isPresent()) { return candidateFine; } current++; } return Optional.empty(); }
@Override Map<Cell, byte[]> populateCells(PartitionInfo info, List<WriteInfo> writes) { SweepableTimestampsTable.SweepableTimestampsRow row = computeRow(info); SweepableTimestampsTable.SweepableTimestampsColumn col = computeColumn(info); SweepableTimestampsTable.SweepableTimestampsColumnValue colVal = SweepableTimestampsTable.SweepableTimestampsColumnValue.of(col, DUMMY); return ImmutableMap.of(SweepQueueUtils.toCell(row, colVal), colVal.persistValue()); }
default List<WriteInfo> toWriteInfos(Map<TableReference, ? extends Map<Cell, byte[]>> writes, long timestamp) { return writes.entrySet().stream() .flatMap(entry -> entry.getValue().entrySet().stream() .map(singleWrite -> SweepQueueUtils.toWriteInfo(entry.getKey(), singleWrite, timestamp))) .collect(Collectors.toList()); }
private SweepableTimestampsTable.SweepableTimestampsColumn computeColumn(PartitionInfo info) { return SweepableTimestampsTable.SweepableTimestampsColumn.of(SweepQueueUtils.tsPartitionFine(info.timestamp())); }
@Test public void deletesGetBatched() { TargetedSweeper sweeperConservative = getSingleShardSweeper(); int numberOfTimestamps = 5 * BATCH_SIZE_KVS / MAX_CELLS_GENERIC + 1; commitTransactionsWithWritesIntoUniqueCells(numberOfTimestamps, MAX_CELLS_GENERIC, sweeperConservative); sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, times(6)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getAllValues().stream().map(Map::size).mapToInt(x -> x).sum()) .isEqualTo(5 * BATCH_SIZE_KVS + MAX_CELLS_GENERIC); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(maxTsForFinePartition(0)); }
private SweepableTimestampsTable.SweepableTimestampsRow computeRow(PartitionInfo partitionInfo) { return SweepableTimestampsTable.SweepableTimestampsRow.of( partitionInfo.shard(), SweepQueueUtils.tsPartitionCoarse(partitionInfo.timestamp()), partitionInfo.isConservative().persistToBytes()); }
private Optional<Long> nextSweepablePartition(ShardAndStrategy shardAndStrategy, long minFineInclusive, long maxFineInclusive) { ColumnRangeSelection range = getColRangeSelection(minFineInclusive, maxFineInclusive + 1); long current = SweepQueueUtils.partitionFineToCoarse(minFineInclusive); long maxCoarseInclusive = SweepQueueUtils.partitionFineToCoarse(maxFineInclusive); while (current <= maxCoarseInclusive) { Optional<Long> candidateFine = getCandidatesInCoarsePartition(shardAndStrategy, current, range); if (candidateFine.isPresent()) { return candidateFine; } current++; } return Optional.empty(); }
private Map<Cell, byte[]> addCell(PartitionInfo info, WriteReference writeRef, boolean isDedicatedRow, long dedicatedRowNumber, long writeIndex) { SweepableCellsRow row = computeRow(info, isDedicatedRow, dedicatedRowNumber); SweepableCellsColumnValue colVal = createColVal(info.timestamp(), writeIndex, writeRef); return ImmutableMap.of(SweepQueueUtils.toCell(row, colVal), colVal.persistValue()); }
default List<WriteInfo> toWriteInfos(Map<TableReference, ? extends Map<Cell, byte[]>> writes, long timestamp) { return writes.entrySet().stream() .flatMap(entry -> entry.getValue().entrySet().stream() .map(singleWrite -> SweepQueueUtils.toWriteInfo(entry.getKey(), singleWrite, timestamp))) .collect(Collectors.toList()); }
private long getTimestampOrPartition(PartitionInfo info, boolean isDedicatedRow) { return isDedicatedRow ? info.timestamp() : SweepQueueUtils.tsPartitionFine(info.timestamp()); }
@Test public void canReadNextTimestampWhenSweepTimestampInSamePartitionAndGreater() { immutableTs = maxTsForFinePartition(TS_FINE_PARTITION); assertThat(tsPartitionFine(getSweepTsCons())).isEqualTo(TS_FINE_PARTITION); assertThat(getSweepTsCons()).isGreaterThan(TS); assertThat(readConservative(shardCons)).contains(TS_FINE_PARTITION); }
@Test public void canReadNextWhenOtherShardsAndStrategiesProgressToEndOfPartitionForConservative() { progress.updateLastSweptTimestamp(thorough(shardCons), maxTsForFinePartition(TS_FINE_PARTITION)); progress.updateLastSweptTimestamp(conservative(shardThor), maxTsForFinePartition(TS_FINE_PARTITION)); progress.updateLastSweptTimestamp(thorough(shardThor), maxTsForFinePartition(TS_FINE_PARTITION)); assertThat(readConservative(shardCons)).contains(TS_FINE_PARTITION); }
private long exactColumnOrElseBeginningOfRow(long startTsInclusive, long partitionFine) { return Math.max(startTsInclusive - SweepQueueUtils.minTsForFinePartition(partitionFine), 0); } }