public static WriteInfo toWriteInfo(TableReference tableRef, Map.Entry<Cell, byte[]> write, long timestamp) { Cell cell = write.getKey(); boolean isTombstone = Arrays.equals(write.getValue(), PtBytes.EMPTY_BYTE_ARRAY); return WriteInfo.of(WriteReference.of(tableRef, cell, isTombstone), timestamp); }
private Map<TableReference, Map<Cell, Long>> writesPerTable(Collection<WriteInfo> writes, Sweeper sweeper) { return writes.stream().collect(Collectors.groupingBy( WriteInfo::tableRef, Collectors.toMap(WriteInfo::cell, write -> write.timestampToDeleteAtExclusive(sweeper)))); } }
@Override public Collection<WriteInfo> filter(Collection<WriteInfo> writeInfos) { Set<TableReference> tablesToCareAbout = getConservativeTables( writeInfos.stream().map(WriteInfo::tableRef).collect(Collectors.toSet())); if (tablesToCareAbout.isEmpty()) { return writeInfos; } Map<TableReference, Long> truncationTimes = watermarkStore.getWatermarks(tablesToCareAbout); return writeInfos.stream() .filter(write -> !truncationTimes.containsKey(write.tableRef()) || write.timestamp() > truncationTimes.get(write.tableRef())) .collect(Collectors.toList()); }
private Map<Cell, byte[]> addWrite(PartitionInfo info, WriteInfo write, boolean dedicate, long index) { return addCell(info, write.writeRef(), dedicate, index / SweepQueueUtils.MAX_CELLS_DEDICATED, index % SweepQueueUtils.MAX_CELLS_DEDICATED); }
private Map<Integer, Integer> enqueueAtLeastThresholdWritesInDefaultShardWithStartTs(long threshold, long startTs) { List<WriteInfo> writeInfos = new ArrayList<>(); int counter = 0; while (writeInfos.stream().filter(write -> write.toShard(DEFAULT_SHARDS) == CONS_SHARD).count() < threshold) { writeInfos.addAll(generateHundredWrites(counter++, startTs)); } sweepQueue.enqueue(writeInfos); return writeInfos.stream() .collect(Collectors.toMap(write -> write.toShard(DEFAULT_SHARDS), write -> 1, (fst, snd) -> fst + snd)); }
WriteBatch add(List<WriteInfo> writeInfos) { writeInfos.forEach(info -> writesByStartTs.put(info.timestamp(), info)); return this; } }
@VisibleForTesting TableMetadataPersistence.SweepStrategy getStrategy(WriteInfo writeInfo) { return cache.getUnchecked(writeInfo.tableRef()); }
default TableReference tableRef() { return writeRef().tableRef(); }
@Test public void partitionWritesByShardStrategyTimestampGroupsOnShardClash() { List<WriteInfo> writes = new ArrayList<>(); for (int i = 0; i <= numShards; i++) { writes.add(getWriteInfoWithFixedCellHash(CONSERVATIVE, i)); } Map<PartitionInfo, List<WriteInfo>> partitions = partitioner.partitionWritesByShardStrategyTimestamp(writes); assertThat(partitions.keySet()) .containsExactly(PartitionInfo.of(writes.get(0).toShard(numShards), true, 1L)); assertThat(Iterables.getOnlyElement(partitions.values())).isEqualTo(writes); }
@Override public Collection<WriteInfo> filter(Collection<WriteInfo> writeInfos) { Set<TableReference> tablesToCareAbout = getConservativeTables( writeInfos.stream().map(WriteInfo::tableRef).collect(Collectors.toSet())); if (tablesToCareAbout.isEmpty()) { return writeInfos; } Map<TableReference, Long> truncationTimes = watermarkStore.getWatermarks(tablesToCareAbout); return writeInfos.stream() .filter(write -> !truncationTimes.containsKey(write.tableRef()) || write.timestamp() > truncationTimes.get(write.tableRef())) .collect(Collectors.toList()); }
WriteBatch add(List<WriteInfo> writeInfos) { writeInfos.forEach(info -> writesByStartTs.put(info.timestamp(), info)); return this; } }
@VisibleForTesting TableMetadataPersistence.SweepStrategy getStrategy(WriteInfo writeInfo) { return cache.getUnchecked(writeInfo.tableRef()); }
static WriteInfo tombstone(TableReference tableRef, Cell cell, long timestamp) { return WriteInfo.of(WriteReference.of(tableRef, cell, true), timestamp); }