@Override public void deleteRange(TableReference tableRef, RangeRequest range) { try (ClosableIterator<RowResult<Set<Long>>> iterator = getRangeOfTimestamps(tableRef, range, AtlasDbConstants.MAX_TS)) { while (iterator.hasNext()) { RowResult<Set<Long>> rowResult = iterator.next(); Multimap<Cell, Long> cellsToDelete = HashMultimap.create(); rowResult.getCells().forEach(entry -> cellsToDelete.putAll(entry.getKey(), entry.getValue())); delete(tableRef, cellsToDelete); } } }
@Override protected BiMap<TableReference, TableReference> readTableMap() { BiMap<TableReference, TableReference> ret = HashBiMap.create(); ClosableIterator<RowResult<Value>> range = kv.getRange( AtlasDbConstants.NAMESPACE_TABLE, RangeRequest.builder().build(), Long.MAX_VALUE); try { while (range.hasNext()) { RowResult<Value> row = range.next(); String shortName = PtBytes.toString(row.getColumns() .get(AtlasDbConstants.NAMESPACE_SHORT_COLUMN_BYTES).getContents()); TableReference ref = getTableRefFromBytes(row.getRowName()); ret.put(ref, TableReference.createWithEmptyNamespace(shortName)); } } finally { range.close(); } return ret; }
default <U> ClosableIterator<U> flatMap(Function<T, Collection<U>> mapper) { return ClosableIterators.wrap(stream().flatMap(obj -> mapper.apply(obj).stream()).iterator(), this); }
public static void run(KeyValueService kvs, PrintWriter output, int batchSize) { Context context = new Context(kvs, output, batchSize); // This potentially needs to iterate multiple times because getRange // only grabs the most recent timestamp for each cell, but we care // about every timestamp in the old scrub queue (this was the main // problem with the old scrub queue). Each iteration will peel off the // top version of each cell until the entire queue is drained. for (int i = 0;; i++) { output.println("Starting iteration " + i + " of scrub migration."); Stopwatch watch = Stopwatch.createStarted(); try (ClosableIterator<RowResult<Value>> iter = kvs.getRange( AtlasDbConstants.OLD_SCRUB_TABLE, RangeRequest.all(), Long.MAX_VALUE)) { if (!iter.hasNext()) { output.println("Finished all iterations of scrub migration."); break; } runOnce(context, iter); } output.println("Finished iteration " + i + " of scrub migration in " + watch.elapsed(TimeUnit.SECONDS) + " seconds."); } }
public void close() { if (currentResults != null) { currentResults.close(); currentResults = null; lastToken = null; } }
private void checkThatTableIsNowOnly(byte[]... rows) { List<byte[]> keys = Lists.newArrayList(); keyValueService.getRange(TEST_TABLE, RangeRequest.all(), AtlasDbConstants.MAX_TS) .forEachRemaining(row -> keys.add(row.getRowName())); assertTrue(Arrays.deepEquals(keys.toArray(), rows)); }
@Test(expected = RuntimeException.class) public void testGetRangeOfTimestampsThrowsOnError() { keyValueService.getRangeOfTimestamps(TEST_NONEXISTING_TABLE, RangeRequest.all(), AtlasDbConstants.MAX_TS) .hasNext(); }
@Override public void close() { delegate().close(); }
private void checkThatTableIsNowOnly(byte[]... rows) { List<byte[]> keys = Lists.newArrayList(); keyValueService.getRange(TEST_TABLE, RangeRequest.all(), AtlasDbConstants.MAX_TS) .forEachRemaining(row -> keys.add(row.getRowName())); assertTrue(Arrays.deepEquals(keys.toArray(), rows)); }
@Override public void deleteRange(TableReference tableRef, RangeRequest range) { try (ClosableIterator<RowResult<Set<Long>>> iterator = getRangeOfTimestamps(tableRef, range, AtlasDbConstants.MAX_TS)) { while (iterator.hasNext()) { RowResult<Set<Long>> rowResult = iterator.next(); Multimap<Cell, Long> cellsToDelete = HashMultimap.create(); for (Entry<Cell, Set<Long>> entry : rowResult.getCells()) { cellsToDelete.putAll(entry.getKey(), entry.getValue()); } delete(tableRef, cellsToDelete); } } }
private void testGetRangeWithTimestamps(boolean reverse) { putTestDataForMultipleTimestamps(); final RangeRequest range; if (!reverse) { range = RangeRequest.builder().startRowInclusive(row(0)).endRowExclusive(row(1)).build(); } else { range = RangeRequest.reverseBuilder().startRowInclusive(row(0)).build(); } ClosableIterator<RowResult<Set<Long>>> rangeWithHistory = keyValueService.getRangeOfTimestamps( TEST_TABLE, range, TEST_TIMESTAMP + 2); RowResult<Set<Long>> row = rangeWithHistory.next(); assertFalse(rangeWithHistory.hasNext()); rangeWithHistory.close(); assertEquals(1, Iterables.size(row.getCells())); Entry<Cell, Set<Long>> cell0 = row.getCells().iterator().next(); assertEquals(2, cell0.getValue().size()); assertTrue(cell0.getValue().contains(TEST_TIMESTAMP)); assertTrue(cell0.getValue().contains(TEST_TIMESTAMP + 1)); }
@Test public void testGetRangeThrowsOnError() { try { keyValueService.getRange(TEST_NONEXISTING_TABLE, RangeRequest.all(), AtlasDbConstants.MAX_TS).hasNext(); Assert.fail("getRange must throw on failure"); } catch (RuntimeException e) { // Expected } }
private Object getSingleRangeInner(ConsecutiveNarrowTable table, int sliceSize) { RangeRequest request = Iterables.getOnlyElement(table.getRangeRequests(1, sliceSize, false)); int startRow = Ints.fromByteArray(request.getStartInclusive()); ClosableIterator<RowResult<Value>> result = table.getKvs().getRange(table.getTableRef(), request, Long.MAX_VALUE); ArrayList<RowResult<Value>> list = Lists.newArrayList(result); result.close(); Preconditions.checkState(list.size() == sliceSize, "List size %s != %s", sliceSize, list.size()); list.forEach(rowResult -> { byte[] rowName = rowResult.getRowName(); int rowNumber = Ints.fromByteArray(rowName); Preconditions.checkState(rowNumber - startRow < sliceSize, "Start Row %s, row number %s, sliceSize %s", startRow, rowNumber, sliceSize); }); return result; }
private Set<Cell> getAllCells(TableReference tableRef) { try (ClosableIterator<RowResult<Value>> iterator = kvs.get() .getRange(tableRef, RangeRequest.all(), Long.MAX_VALUE)) { return iterator.stream() .map(RowResult::getCells) .flatMap(Streams::stream) .map(Map.Entry::getKey) .collect(Collectors.toSet()); } }
private static Multimap<Cell, Long> doGetAllTimestamps(DbReadTable table, Iterable<Cell> cells, long timestamp) { try (ClosableIterator<AgnosticLightResultRow> iter = table.getAllCells(cells, timestamp, false)) { Multimap<Cell, Long> results = ArrayListMultimap.create(); while (iter.hasNext()) { AgnosticLightResultRow row = iter.next(); Cell cell = Cell.create(row.getBytes(ROW), row.getBytes(COL)); long ts = row.getLong(TIMESTAMP); results.put(cell, ts); } return results; } }
rangeRequest, TEST_TIMESTAMP + 1); assertFalse(keyValueService.getRange(TEST_TABLE, rangeRequest, TEST_TIMESTAMP).hasNext()); assertTrue(rangeResult.hasNext()); assertEquals( RowResult.create( column(2), Value.create(val(1, 2), TEST_TIMESTAMP)).build()), rangeResult.next()); assertTrue(rangeResult.hasNext()); assertEquals( RowResult.create( column(2), Value.create(val(2, 2), TEST_TIMESTAMP)).build()), rangeResult.next()); rangeResult.close();
@Test public void testDeleteMultipleVersions() { putTestDataForMultipleTimestamps(); ClosableIterator<RowResult<Value>> result = keyValueService.getRange( TEST_TABLE, RangeRequest.all(), TEST_TIMESTAMP + 1); assertTrue(result.hasNext()); keyValueService.delete(TEST_TABLE, ImmutableMultimap.of(TEST_CELL, TEST_TIMESTAMP)); result = keyValueService.getRange(TEST_TABLE, RangeRequest.all(), TEST_TIMESTAMP + 1); assertFalse(result.hasNext()); result = keyValueService.getRange(TEST_TABLE, RangeRequest.all(), TEST_TIMESTAMP + 2); assertTrue(result.hasNext()); }
@Override protected <K extends Exception> void batchAcceptSizeHint( int batchSizeHint, ConsistentVisitor<SortedMap<Long, Multimap<TableReference, Cell>>, K> visitor) throws K { ClosableIterator<RowResult<Value>> iterator = getIteratorToScrub(batchSizeHint, maxScrubTimestamp, startRow, endRow); try { BatchingVisitableFromIterable.create(iterator).batchAccept( batchSizeHint, batch -> visitor.visitOne(transformRows(batch))); } finally { iterator.close(); } } };