private WriteInfo getWriteAt(long timestamp) { return WriteInfo.write(TABLE_REF, CELL, timestamp); }
private static WriteInfo writeInfo(long timestamp) { return WriteInfo.write(TABLE, CELL, timestamp); }
List<WriteInfo> writeToCellsInFixedShard(SweepQueueTable writer, long ts, int number, TableReference tableRef) { List<WriteInfo> result = new ArrayList<>(); for (long i = 0; i < number; i++) { Cell cell = getCellWithFixedHash(i); result.add(WriteInfo.write(tableRef, cell, ts)); } putTimestampIntoTransactionTable(ts, ts); writer.enqueue(result); return result; }
private List<WriteInfo> generateHundredWrites(int startCol, long startTs) { List<WriteInfo> writeInfos = new ArrayList<>(); for (int i = 0; i < 100; i++) { writeInfos.add(WriteInfo.write(TABLE_CONS, Cell.create(DEFAULT_CELL.getRowName(), PtBytes.toBytes(startCol * 100 + i)), startTs)); } return writeInfos; } }
private void assertSweepableCellsHasEntryForTimestamp(long timestamp) { SweepBatch batch = sweepableCells.getBatchForPartition( ShardAndStrategy.conservative(CONS_SHARD), tsPartitionFine(timestamp), -1L, timestamp + 1); assertThat(batch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, timestamp)); }
@Test public void readDoesNotReturnValuesFromAbortedTransactions() { writeToDefaultCellAborted(sweepableCells, TS + 1, TABLE_CONS); SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS)); }
private List<WriteInfo> writeWithoutCommitConservative(long timestamp, long row, long numWrites) { List<WriteInfo> writes = new ArrayList<>(); for (long i = 0; i < numWrites; i++) { Cell cell = Cell.create(PtBytes.toBytes(row), PtBytes.toBytes(i)); writes.add(WriteInfo.write(TABLE_CONS, cell, timestamp)); } sweepableCells.enqueue(writes); return writes; }
@Test public void readDoesNotReturnValuesFromTransactionsCommittedAfterSweepTs() { writeToDefaultCellCommitedAt(sweepableCells, TS + 1, SMALL_SWEEP_TS, TABLE_CONS); SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS)); }
@Test public void readDoesNotReturnValuesFromUncommittedTransactionsAndAbortsThem() { writeToDefaultCellUncommitted(sweepableCells, TS + 1, TABLE_CONS); assertThat(!isTransactionAborted(TS + 1)); SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS); assertThat(isTransactionAborted(TS + 1)); assertThat(conservativeBatch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS)); }
@Test public void readOnlyMostRecentTimestampForRange() { writeToDefaultCellCommitted(sweepableCells, TS - 1, TABLE_CONS); writeToDefaultCellCommitted(sweepableCells, TS + 2, TABLE_CONS); writeToDefaultCellCommitted(sweepableCells, TS - 2, TABLE_CONS); writeToDefaultCellCommitted(sweepableCells, TS + 1, TABLE_CONS); SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 3, TS); assertThat(conservativeBatch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS - 1)); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(TS - 1); conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 3, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS + 2)); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(SMALL_SWEEP_TS - 1); }
@Test public void canReadSingleEntryInSingleShardForCorrectPartitionAndRange() { SweepBatch conservativeBatch = readConservative(shardCons, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS); assertThat(conservativeBatch.writes()).containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS)); SweepBatch thoroughBatch = readThorough(TS2_FINE_PARTITION, TS2 - 1, Long.MAX_VALUE); assertThat(thoroughBatch.writes()).containsExactly(WriteInfo.write(TABLE_THOR, DEFAULT_CELL, TS2)); SweepMetricsAssert.assertThat(metricsManager).hasEnqueuedWritesConservativeEqualTo(1); SweepMetricsAssert.assertThat(metricsManager).hasEnqueuedWritesThoroughEqualTo(1); }
@Test public void changingNumberOfShardsDoesNotAffectExistingWritesButAffectsFuture() { useSingleShard(); assertThat(readConservative(0, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS).writes()).isEmpty(); writeToDefaultCellCommitted(sweepableCells, TS, TABLE_CONS); assertThat(readConservative(0, TS_FINE_PARTITION, TS - 1, SMALL_SWEEP_TS).writes()) .containsExactly(WriteInfo.write(TABLE_CONS, DEFAULT_CELL, TS)); }
private void commitTransactionsWithWritesIntoUniqueCells(int transactions, int writes, TargetedSweeper sweeper) { for (int i = 1000; i < 1000 + transactions; i++) { putTimestampIntoTransactionTable(i, i); List<WriteInfo> writeInfos = new ArrayList<>(); for (int j = 0; j < writes; j++) { Cell cell = Cell.create(PtBytes.toBytes(i), PtBytes.toBytes(j)); writeInfos.add(WriteInfo.write(TABLE_CONS, cell, i)); } sweeper.enqueue(writeInfos); } }
private WriteInfo getWriteInfo(TableReference tableRef, int rowIndex, int colIndex, long timestamp) { Cell cell = Cell.create(PtBytes.toBytes(rowIndex), PtBytes.toBytes(colIndex)); return WriteInfo.write(tableRef, cell, timestamp); } }
@Test public void canReadMultipleEntriesInSingleShardDifferentTransactions() { writeToCellCommitted(sweepableCells, TS, getCellWithFixedHash(1), TABLE_CONS); writeToCellCommitted(sweepableCells, TS + 1, getCellWithFixedHash(2), TABLE_CONS); SweepBatch conservativeBatch = readConservative(FIXED_SHARD, TS_FINE_PARTITION, TS - 1, TS + 2); assertThat(conservativeBatch.writes()).containsExactlyInAnyOrder( WriteInfo.write(TABLE_CONS, getCellWithFixedHash(1), TS), WriteInfo.write(TABLE_CONS, getCellWithFixedHash(2), TS + 1)); assertThat(conservativeBatch.lastSweptTimestamp()).isEqualTo(TS + 1); }
@Test public void doNotMissSingleWriteInNextIteration() { TargetedSweeper sweeperConservative = getSingleShardSweeper(); int minTsToReachBatchSize = (SWEEP_BATCH_SIZE - 1) / MAX_CELLS_GENERIC + 1; commitTransactionsWithWritesIntoUniqueCells(minTsToReachBatchSize, MAX_CELLS_GENERIC, sweeperConservative); // put one additional transaction with a single write after putTimestampIntoTransactionTable(1000 + minTsToReachBatchSize, 1000 + minTsToReachBatchSize); Cell cell = Cell.create(PtBytes.toBytes(1000 + minTsToReachBatchSize), PtBytes.toBytes(0)); sweeperConservative.enqueue(ImmutableList.of(WriteInfo.write(TABLE_CONS, cell, 1000 + minTsToReachBatchSize))); // first iteration of sweep should include all but one of the writes, since deletes are batched, we do not // specify the number of calls to delete sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); ArgumentCaptor<Map> map = ArgumentCaptor.forClass(Map.class); verify(spiedKvs, atLeast(1)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getAllValues().stream().map(Map::size).mapToInt(x -> x).sum()) .isEqualTo(SWEEP_BATCH_SIZE); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(1000 + minTsToReachBatchSize - 1); // second iteration of sweep should contain the remaining write sweeperConservative.sweepNextBatch(ShardAndStrategy.conservative(0)); verify(spiedKvs, atLeast(2)).deleteAllTimestamps(eq(TABLE_CONS), map.capture(), eq(false)); assertThat(map.getValue().size()).isEqualTo(1); assertThat(progress.getLastSweptTimestamp(ShardAndStrategy.conservative(0))) .isEqualTo(maxTsForFinePartition(0)); }