@Override protected void put(final TableReference tableRef, Cell cell, final String val, final long ts) { super.put(tableRef, cell, val, ts); sweepQueue.enqueue(ImmutableMap.of(tableRef, ImmutableMap.of(cell, PtBytes.toBytes(val))), ts); }
private List<WriteInfo> getEnqueuedWritesNumber(int index) { ArgumentCaptor<List> writes = ArgumentCaptor.forClass(List.class); verify(sweepQueue, atLeast(index)).enqueue(writes.capture()); return writes.getAllValues().get(index - 1); }
private void enqueueWriteUncommitted(TableReference tableRef, long startTs) { sweepQueue.enqueue(writeToDefaultCell(tableRef, startTs), startTs); }
private Map<Integer, Integer> enqueueAtLeastThresholdWritesInDefaultShardWithStartTs(long threshold, long startTs) { List<WriteInfo> writeInfos = new ArrayList<>(); int counter = 0; while (writeInfos.stream().filter(write -> write.toShard(DEFAULT_SHARDS) == CONS_SHARD).count() < threshold) { writeInfos.addAll(generateHundredWrites(counter++, startTs)); } sweepQueue.enqueue(writeInfos); return writeInfos.stream() .collect(Collectors.toMap(write -> write.toShard(DEFAULT_SHARDS), write -> 1, (fst, snd) -> fst + snd)); }
private void enqueueTombstone(TableReference tableRef, long ts) { putTimestampIntoTransactionTable(ts, ts); sweepQueue.enqueue(tombstoneToDefaultCell(tableRef, ts), ts); }
@Test public void writesNotAddedToSweepQueueOrKvsOnException() { assertThatThrownBy(() -> txManager.runTaskWithRetry(txn -> { txn.put(TABLE_CONS, ImmutableMap.of(TEST_CELL, TEST_DATA)); throw new RuntimeException("test"); })).isInstanceOf(RuntimeException.class); verify(sweepQueue, times(0)).enqueue(anyList()); assertNoEntryForCellInKvs(TABLE_CONS, TEST_CELL); }
@Test public void callingEnqueueAndSweepOnUninitializedSweeperThrows() { TargetedSweeper uninitializedSweeper = TargetedSweeper.createUninitializedForTest(null); assertThatThrownBy(() -> uninitializedSweeper.enqueue(ImmutableList.of())) .isInstanceOf(NotInitializedException.class) .hasMessageContaining("Targeted Sweeper"); assertThatThrownBy(() -> uninitializedSweeper.sweepNextBatch(ShardAndStrategy.conservative(0))) .isInstanceOf(NotInitializedException.class) .hasMessageContaining("Targeted Sweeper"); }
@Test public void writesAddedToSweepQueueAndKvsOnPreCommitConditionFailure() { long startTs = putWriteAndFailOnPreCommitConditionReturningStartTimestamp(SINGLE_WRITE); assertThat(getEnqueuedWritesNumber(1)).containsExactly(WriteInfo.of(SINGLE_WRITE, startTs)); verify(sweepQueue, times(1)).enqueue(anyList()); assertLatestEntryForCellInKvsAtTimestamp(TABLE_CONS, TEST_CELL, startTs); }
private void commitTransactionsWithWritesIntoUniqueCells(int transactions, int writes, TargetedSweeper sweeper) { for (int i = 1000; i < 1000 + transactions; i++) { putTimestampIntoTransactionTable(i, i); List<WriteInfo> writeInfos = new ArrayList<>(); for (int j = 0; j < writes; j++) { Cell cell = Cell.create(PtBytes.toBytes(i), PtBytes.toBytes(j)); writeInfos.add(WriteInfo.write(TABLE_CONS, cell, i)); } sweeper.enqueue(writeInfos); } }
@Test public void committedWritesAreAddedToSweepQueue() { List<WriteReference> table1Writes = ImmutableList.of( WriteReference.write(TABLE_CONS, Cell.create("a".getBytes(), "b".getBytes())), WriteReference.write(TABLE_CONS, Cell.create("a".getBytes(), "c".getBytes())), WriteReference.tombstone(TABLE_CONS, Cell.create("a".getBytes(), "d".getBytes())), WriteReference.write(TABLE_CONS, Cell.create("b".getBytes(), "d".getBytes()))); List<WriteReference> table2Writes = ImmutableList.of( WriteReference.write(TABLE_THOR, Cell.create("w".getBytes(), "x".getBytes())), WriteReference.write(TABLE_THOR, Cell.create("y".getBytes(), "z".getBytes())), WriteReference.tombstone(TABLE_THOR, Cell.create("z".getBytes(), "z".getBytes()))); long startTimestamp = txManager.runTaskWithRetry(txn -> { table1Writes.forEach(write -> put(txn, write)); table2Writes.forEach(write -> put(txn, write)); return txn.getTimestamp(); }); List<WriteInfo> expectedWrites = Stream.concat(table1Writes.stream(), table2Writes.stream()) .map(writeRef -> WriteInfo.of(writeRef, startTimestamp)) .collect(Collectors.toList()); assertThat(getEnqueuedWritesNumber(1)).hasSameElementsAs(expectedWrites); verify(sweepQueue, times(1)).enqueue(anyList()); }
@Test public void writesNotAddedToSweepQueueOrKvsOnWriteWriteConflict() { Transaction t1 = txManager.createNewTransaction(); Transaction t2 = txManager.createNewTransaction(); put(t1, SINGLE_WRITE); put(t2, SINGLE_WRITE); t1.commit(); assertThatThrownBy(t2::commit).isInstanceOf(TransactionConflictException.class); verify(sweepQueue, times(1)).enqueue(anyList()); assertLatestEntryForCellInKvsAtTimestamp(TABLE_CONS, TEST_CELL, t1.getTimestamp()); }
@Test public void doNotMissSingleWriteInNextIteration() { TargetedSweeper sweeperConservative = getSingleShardSweeper(); int minTsToReachBatchSize = (SWEEP_BATCH_SIZE - 1) / MAX_CELLS_GENERIC + 1; commitTransactionsWithWritesIntoUniqueCells(minTsToReachBatchSize, MAX_CELLS_GENERIC, sweeperConservative); // put one additional transaction with a single write after putTimestampIntoTransactionTable(1000 + minTsToReachBatchSize, 1000 + minTsToReachBatchSize); Cell cell = Cell.create(PtBytes.toBytes(1000 + minTsToReachBatchSize), PtBytes.toBytes(0)); sweeperConservative.enqueue(ImmutableList.of(WriteInfo.write(TABLE_CONS, cell, 1000 + minTsToReachBatchSize))); // first iteration of sweep should include all but one of the writes, since deletes are batched, we do not // specify the number of calls to delete sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, atLeast(1)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getAllValues().stream().map(Map::size).mapToInt(x -> x).sum()) .isEqualTo(SWEEP_BATCH_SIZE); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(1000 + minTsToReachBatchSize - 1); // second iteration of sweep should contain the remaining write sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); verify(spiedKvs, atLeast(2)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getValue().size()).isEqualTo(1); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(maxTsForFinePartition(0)); }
@Test public void writesAddedToSweepQueueOnNoConflict() { WriteReference firstWrite = WriteReference.write(TABLE_CONS, TEST_CELL); WriteReference secondWrite = WriteReference.write(TABLE_THOR, TEST_CELL); Transaction t1 = txManager.createNewTransaction(); Transaction t2 = txManager.createNewTransaction(); put(t1, firstWrite); put(t2, secondWrite); t1.commit(); assertThat(getEnqueuedWritesNumber(1)).containsExactly(WriteInfo.of(firstWrite, t1.getTimestamp())); assertLatestEntryForCellInKvsAtTimestamp(TABLE_CONS, TEST_CELL, t1.getTimestamp()); t2.commit(); assertThat(getEnqueuedWritesNumber(2)).containsExactly(WriteInfo.of(secondWrite, t2.getTimestamp())); assertLatestEntryForCellInKvsAtTimestamp(TABLE_THOR, TEST_CELL, t2.getTimestamp()); verify(sweepQueue, times(2)).enqueue(anyList()); }
@Override protected void put(final TableReference tableRef, Cell cell, final String val, final long ts) { super.put(tableRef, cell, val, ts); sweepQueue.enqueue(ImmutableMap.of(tableRef, ImmutableMap.of(cell, PtBytes.toBytes(val))), ts); }