CandidateCellForSweepingRequest request) { RangeRequest range = RangeRequest.builder() .startRowInclusive(request.startRowInclusive()) .batchHint(request.batchSizeHint().orElse( AtlasDbConstants.DEFAULT_SWEEP_CANDIDATE_BATCH_HINT)) .build(); try (ReleasableCloseable<ClosableIterator<RowResult<Value>>> valueResults = new ReleasableCloseable<>( getValues(tableRef, range, request.maxTimestampExclusive(), request.shouldCheckIfLatestValueIsEmpty()));
public void migrate() { List<byte[]> rangeBoundaries = getRangeBoundaries(); int totalTasks = rangeBoundaries.size() - 1; progress.beginTask("Migrating table " + srcTable + "...", totalTasks); Map<Long, byte[]> boundaryById = Maps.newHashMap(); for (long rangeId = 0; rangeId < rangeBoundaries.size() - 1; rangeId++) { boundaryById.put(rangeId, rangeBoundaries.get((int) rangeId)); } checkpointer.createCheckpoints(srcTable.getQualifiedName(), boundaryById); // Look up the checkpoints and log start point (or done) rangeMigrator.logStatus(rangeBoundaries.size()); List<Future<Void>> futures = Lists.newArrayList(); for (long rangeId = 0; rangeId < rangeBoundaries.size() - 1; rangeId++) { byte[] end = rangeBoundaries.get((int) rangeId + 1); // the range's start will be set within the transaction RangeRequest range = RangeRequest.builder() .endRowExclusive(end) .batchHint(readBatchSize) .retainColumns(columnSelection) .build(); Callable<Void> task = createMigrationTask( range, rangeId); Callable<Void> wrappedTask = PTExecutors.wrap(task); Future<Void> future = executor.submit(wrappedTask); futures.add(future); } waitForFutures(futures); progress.taskComplete(); }
EncodingUtils.flipAllBitsInPlace(startRow); RangeRequest rangeRequest = RangeRequest.builder() .startRowInclusive(startRow) .retainColumns(ImmutableList.of(COLUMN)) .batchHint(1000) .build();
@Test public void testKeyValueRangeColumnSelection() { putDirect("row1", "col1", "v1", 0); putDirect("row1", "col2", "v2", 2); putDirect("row1", "col4", "v5", 3); putDirect("row1a", "col4", "v5", 100); putDirect("row2", "col2", "v3", 1); putDirect("row2", "col4", "v4", 6); List<byte[]> selectedColumns = ImmutableList.of(PtBytes.toBytes("col2")); RangeRequest simpleRange = RangeRequest.builder().retainColumns(ColumnSelection.create(selectedColumns)).build(); ImmutableList<RowResult<Value>> list = ImmutableList.copyOf(keyValueService.getRange(TEST_TABLE, simpleRange, 1)); assertEquals(0, list.size()); list = ImmutableList.copyOf(keyValueService.getRange(TEST_TABLE, simpleRange, 2)); assertEquals(1, list.size()); RowResult<Value> row = list.iterator().next(); assertEquals(1, row.getColumns().size()); list = ImmutableList.copyOf(keyValueService.getRange(TEST_TABLE, simpleRange, 3)); assertEquals(2, list.size()); row = list.iterator().next(); assertEquals(1, row.getColumns().size()); list = ImmutableList.copyOf(keyValueService.getRange(TEST_TABLE, simpleRange.getBuilder().endRowExclusive(PtBytes.toBytes("row2")).build(), 3)); assertEquals(1, list.size()); row = list.iterator().next(); assertEquals(1, row.getColumns().size()); list = ImmutableList.copyOf(keyValueService.getRange(TEST_TABLE, simpleRange.getBuilder().startRowInclusive(PtBytes.toBytes("row1a")).build(), 3)); assertEquals(1, list.size()); row = list.iterator().next(); assertEquals(1, row.getColumns().size()); }
private void verifyRanges(Transaction readOnlyTransaction) { // verify each set of reads to ensure they are the same. for (Entry<TableReference, ConcurrentMap<RangeRequest, byte[]>> tableAndRange : rangeEndByTable.entrySet()) { TableReference table = tableAndRange.getKey(); Map<RangeRequest, byte[]> rangeEnds = tableAndRange.getValue(); for (Entry<RangeRequest, byte[]> rangeAndRangeEndEntry : rangeEnds.entrySet()) { RangeRequest range = rangeAndRangeEndEntry.getKey(); byte[] rangeEnd = rangeAndRangeEndEntry.getValue(); if (rangeEnd.length != 0 && !RangeRequests.isTerminalRow(range.isReverse(), rangeEnd)) { range = range.getBuilder() .endRowExclusive(RangeRequests.getNextStartRow(range.isReverse(), rangeEnd)) .build(); } ConcurrentNavigableMap<Cell, byte[]> writes = writesByTable.get(table); BatchingVisitableView<RowResult<byte[]>> bv = BatchingVisitableView.of( readOnlyTransaction.getRange(table, range)); NavigableMap<Cell, ByteBuffer> readsInRange = Maps.transformValues( getReadsInRange(table, range), ByteBuffer::wrap); if (!bv.transformBatch(input -> filterWritesFromRows(input, writes)).isEqual(readsInRange.entrySet())) { handleTransactionConflict(table); } } } }
@Test public void testRowRangeSecondColumnFirstTwoResults() { AbstractTransaction transaction = mock(AbstractTransaction.class); Cell expectedCell = getCell(TEST_ROW_KEY, SECOND_COL_SHORT_NAME); Cell anotherExpectedCell = getCell(TEST_ROW_KEY2, SECOND_COL_SHORT_NAME); Cell cellToBeDroppedFromResults = getCell(TEST_ROW_KEY3, SECOND_COL_SHORT_NAME); RangeRequest expectedRange = RangeRequest.builder() .startRowInclusive(TEST_ROW_KEY.getBytes()) .endRowExclusive(RANGE_END_ROW_KEY.getBytes()) .retainColumns(SECOND_COLUMN_SELECTION) .batchHint(2) .build(); when(transaction.getRange(tableRef, expectedRange)).thenReturn( BatchingVisitableFromIterable.create(Arrays.asList( RowResult.of(expectedCell, STRING_VALUE_PERSISTER.persistToBytes(TEST_VALUE_STRING)), RowResult.of(anotherExpectedCell, STRING_VALUE_PERSISTER.persistToBytes(TEST_VALUE_STRING2)), RowResult.of(cellToBeDroppedFromResults, STRING_VALUE_PERSISTER.persistToBytes(TEST_VALUE_STRING3)) )) ); Map<String, StringValue> result = getRangeSecondColumnOnlyFirstTwoResults(transaction, TEST_ROW_KEY, RANGE_END_ROW_KEY); assertThat(result) .isEqualTo(ImmutableMap.of(TEST_ROW_KEY, TEST_VALUE_STRING, TEST_ROW_KEY2, TEST_VALUE_STRING2)); verify(transaction, times(1)).getRange(tableRef, expectedRange); }
@Override public RangeToken getRange(TransactionToken token, final TableRange range) { return runReadOnly(token, transaction -> { int limit = range.getBatchSize() + 1; RangeRequest request = RangeRequest.builder() .startRowInclusive(range.getStartRow()) .endRowExclusive(range.getEndRow()) .batchHint(limit) .retainColumns(range.getColumns()) .build(); BatchingVisitable<RowResult<byte[]>> visitable = transaction.getRange(getTableRef(range.getTableName()), request); List<RowResult<byte[]>> results = BatchingVisitables.limit(visitable, limit).immutableCopy(); if (results.size() == limit) { TableRowResult data = new TableRowResult(range.getTableName(), results.subList(0, limit - 1)); RowResult<byte[]> lastResultInBatch = results.get(limit - 1); TableRange nextRange = range.withStartRow(lastResultInBatch.getRowName()); return new RangeToken(data, nextRange); } else { TableRowResult data = new TableRowResult(range.getTableName(), results); return new RangeToken(data, null); } }); }
private void doTestGetRangePagingLastRowEdgeCase(int numColumnsInMetadata, int batchSizeHint, boolean reverse) { TableReference tableRef = createTableWithNamedColumns(numColumnsInMetadata); byte[] last = reverse ? RangeRequests.getFirstRowName() : RangeRequests.getLastRowName(); Map<Cell, byte[]> values = ImmutableMap.of( Cell.create(last, PtBytes.toBytes("c1")), PtBytes.toBytes("a"), Cell.create(last, last), PtBytes.toBytes("b")); keyValueService.put(tableRef, values, TEST_TIMESTAMP); RangeRequest request = RangeRequest.builder(reverse).batchHint(batchSizeHint).build(); try (ClosableIterator<RowResult<Value>> iter = keyValueService.getRange(tableRef, request, Long.MAX_VALUE)) { List<RowResult<Value>> results = ImmutableList.copyOf(iter); List<RowResult<Value>> expected = ImmutableList.of( RowResult.create(last, ImmutableSortedMap.<byte[], Value>orderedBy(UnsignedBytes.lexicographicalComparator()) .put(PtBytes.toBytes("c1"), Value.create(PtBytes.toBytes("a"), TEST_TIMESTAMP)) .put(last, Value.create(PtBytes.toBytes("b"), TEST_TIMESTAMP)) .build())); assertEquals(expected, results); } }
@Test public void testRowRangeSecondColumn() { AbstractTransaction transaction = mock(AbstractTransaction.class); Cell expectedCell = getCell(TEST_ROW_KEY, SECOND_COL_SHORT_NAME); Cell anotherExpectedCell = getCell(TEST_ROW_KEY2, SECOND_COL_SHORT_NAME); RangeRequest expectedRange = RangeRequest.builder() .startRowInclusive(TEST_ROW_KEY.getBytes()) .endRowExclusive(RANGE_END_ROW_KEY.getBytes()) .retainColumns(SECOND_COLUMN_SELECTION) .build(); when(transaction.getRange(tableRef, expectedRange)).thenReturn( BatchingVisitableFromIterable.create(Arrays.asList( RowResult.of(expectedCell, STRING_VALUE_PERSISTER.persistToBytes(TEST_VALUE_STRING)), RowResult.of(anotherExpectedCell, STRING_VALUE_PERSISTER.persistToBytes(TEST_VALUE_STRING2)) )) ); Map<String, StringValue> result = getRangeSecondColumn(transaction, TEST_ROW_KEY, RANGE_END_ROW_KEY); assertThat(result) .isEqualTo(ImmutableMap.of(TEST_ROW_KEY, TEST_VALUE_STRING, TEST_ROW_KEY2, TEST_VALUE_STRING2)); verify(transaction, times(1)).getRange(tableRef, expectedRange); }
public Iterable<RangeRequest> getRangeRequests(int numRequests, int sliceSize, boolean allColumns) { List<RangeRequest> requests = Lists.newArrayList(); Set<Integer> used = Sets.newHashSet(); for (int i = 0; i < numRequests; i++) { int startRow; do { startRow = getRandom().nextInt(getNumRows() - sliceSize); } while (used.contains(startRow)); int endRow = startRow + sliceSize; RangeRequest request = RangeRequest.builder() .batchHint(1 + sliceSize) .startRowInclusive(Ints.toByteArray(startRow)) .endRowExclusive(Ints.toByteArray(endRow)) .retainColumns(allColumns ? ColumnSelection.all() : ColumnSelection.create(ImmutableList.of(Tables.COLUMN_NAME_IN_BYTES.array()))) .build(); requests.add(request); used.add(startRow); } return requests; }
private byte[] copyOneTransactionInternal(RangeRequest range, long rangeId, Transaction readT, Transaction writeT) { final long maxBytes = TransactionConstants.WARN_LEVEL_FOR_QUEUED_BYTES / 2; byte[] start = getCheckpoint(rangeId, writeT); if (start == null) { return null; } RangeRequest.Builder builder = range.getBuilder().startRowInclusive(start); if (builder.isInvalidRange()) { return null; } RangeRequest rangeToUse = builder.build(); BatchingVisitable<RowResult<byte[]>> bv = readT.getRange(srcTable, rangeToUse); byte[] lastRow = internalCopyRange(bv, maxBytes, writeT); byte[] nextRow = getNextRowName(lastRow); checkpointer.checkpoint(srcTable.getQualifiedName(), rangeId, nextRow, writeT); return lastRow; }
private void doTestGetRangePagingWithColumnSelection(int batchSizeHint, int numRows, int numColsInSelection, boolean reverse) { Collection<byte[]> columnSelection = new ArrayList<>(numColsInSelection); for (long col = 1; col <= numColsInSelection; ++col) { byte[] colName = PtBytes.toBytes(Long.MIN_VALUE ^ col); columnSelection.add(colName); } RangeRequest request = RangeRequest.builder(reverse) .retainColumns(columnSelection) .batchHint(batchSizeHint) .build(); try (ClosableIterator<RowResult<Value>> iter = keyValueService.getRange(TEST_TABLE, request, TEST_TIMESTAMP + 1)) { List<RowResult<Value>> results = ImmutableList.copyOf(iter); assertEquals(getExpectedResultForRangePagingWithColumnSelectionTest(numRows, numColsInSelection, reverse), results); } }
@Test public void testGetRangesPaging() { Transaction t = startTransaction(); byte[] row0Bytes = PtBytes.toBytes("row0"); byte[] row00Bytes = PtBytes.toBytes("row00"); byte[] colBytes = PtBytes.toBytes("col"); Cell k1 = Cell.create(row00Bytes, colBytes); byte[] row1Bytes = PtBytes.toBytes("row1"); Cell k2 = Cell.create(row1Bytes, colBytes); byte[] v = PtBytes.toBytes("v"); t.put(TEST_TABLE, ImmutableMap.of(Cell.create(row0Bytes, colBytes), v)); t.put(TEST_TABLE, ImmutableMap.of(k1, v)); t.put(TEST_TABLE, ImmutableMap.of(k2, v)); t.commit(); t = startTransaction(); t.delete(TEST_TABLE, ImmutableSet.of(k1)); t.commit(); t = startTransaction(); byte[] rangeEnd = RangeRequests.nextLexicographicName(row00Bytes); List<RangeRequest> ranges = ImmutableList.of(RangeRequest.builder().prefixRange(row0Bytes).endRowExclusive(rangeEnd).batchHint(1).build()); verifyAllGetRangesImplsNumRanges(t, ranges, ImmutableList.of("v")); }
private void testGetRangeWithTimestamps(boolean reverse) { putTestDataForMultipleTimestamps(); final RangeRequest range; if (!reverse) { range = RangeRequest.builder().startRowInclusive(row(0)).endRowExclusive(row(1)).build(); } else { range = RangeRequest.reverseBuilder().startRowInclusive(row(0)).build(); } ClosableIterator<RowResult<Set<Long>>> rangeWithHistory = keyValueService.getRangeOfTimestamps( TEST_TABLE, range, TEST_TIMESTAMP + 2); RowResult<Set<Long>> row = rangeWithHistory.next(); assertFalse(rangeWithHistory.hasNext()); rangeWithHistory.close(); assertEquals(1, Iterables.size(row.getCells())); Entry<Cell, Set<Long>> cell0 = row.getCells().iterator().next(); assertEquals(2, cell0.getValue().size()); assertTrue(cell0.getValue().contains(TEST_TIMESTAMP)); assertTrue(cell0.getValue().contains(TEST_TIMESTAMP + 1)); }
@Test public void testGetRangeOfTimestampsFetchesProperRange() { keyValueService.put(TEST_TABLE, ImmutableMap.of( Cell.create(row(0), column(0)), val(0, 5), Cell.create(row(1), column(0)), val(0, 5), Cell.create(row(2), column(0)), val(0, 5)), TEST_TIMESTAMP); keyValueService.put(TEST_TABLE, ImmutableMap.of( Cell.create(row(0), column(0)), val(0, 7)), TEST_TIMESTAMP + 10); RangeRequest range = RangeRequest.builder().startRowInclusive(row(1)).endRowExclusive(row(2)).build(); List<RowResult<Set<Long>>> results = ImmutableList.copyOf( keyValueService.getRangeOfTimestamps(TEST_TABLE, range, TEST_TIMESTAMP + 1)); assertEquals(1, results.size()); assertArrayEquals(row(1), results.get(0).getRowName()); }
@Override protected Map<String, StringValue> getRangeSecondColumnOnlyFirstTwoResults( Transaction transaction, String startRowKey, String endRowKey) { SchemaApiTestTable table = tableFactory.getSchemaApiTestTable(transaction); ColumnSelection secondColSelection = SchemaApiTestTable.getColumnSelection( SchemaApiTestTable.SchemaApiTestNamedColumn.COLUMN2); RangeRequest rangeRequest = RangeRequest.builder() .startRowInclusive(SchemaApiTestRow.of(startRowKey).persistToBytes()) .endRowExclusive(SchemaApiTestRow.of(endRowKey).persistToBytes()) .retainColumns(secondColSelection) .batchHint(2) .build(); BatchingVisitableView<SchemaApiTestRowResult> rangeRequestResult = table.getRange(rangeRequest); return BatchingVisitables.take(rangeRequestResult, 2) .stream() .collect(Collectors.toMap( entry -> entry.getRowName().getComponent1(), SchemaApiTestTable.SchemaApiTestRowResult::getColumn2)); }
@Test public void testRangesTransactionColumnSelection() { Transaction t = startTransaction(); put(t, "row1", "col1", "v1"); t.commit(); RangeRequest range1 = RangeRequest.builder().batchHint(3).build(); RangeRequest range2 = range1.getBuilder().retainColumns(ColumnSelection.create(ImmutableSet.of(PtBytes.toBytes("col1")))).build(); t = startTransaction(); Iterable<BatchingVisitable<RowResult<byte[]>>> ranges = t.getRanges(TEST_TABLE, Iterables.limit(Iterables.cycle(range1, range2), 1000)); for (BatchingVisitable<RowResult<byte[]>> batchingVisitable : ranges) { final List<RowResult<byte[]>> list = BatchingVisitables.copyToList(batchingVisitable); assertEquals(1, list.size()); assertEquals(1, list.get(0).getColumns().size()); } RangeRequest range3 = range1.getBuilder().retainColumns(ColumnSelection.create(ImmutableSet.of(PtBytes.toBytes("col2")))).build(); verifyAllGetRangesImplsRangeSizes(t, range3, 0); }
private ClosableIterator<RowResult<Value>> getIteratorToScrub( int batchSizeHint, long maxScrubTimestamp, byte[] startRow, byte[] endRow) { RangeRequest.Builder range = RangeRequest.builder(); if (startRow != null) { range = range.startRowInclusive(startRow); } if (endRow != null) { range = range.endRowExclusive(endRow); } return keyValueService.getRange( AtlasDbConstants.SCRUB_TABLE, range.batchHint(batchSizeHint).build(), maxScrubTimestamp); }
@Test public void testRangePagingBatchSizeOneReverse() { if (!supportsReverse()) { return; } int totalPuts = 100; for (int i = 0 ; i < totalPuts ; i++) { putDirect("row"+i, "col1", "v1", 0); } RangeRequest rangeRequest = RangeRequest.reverseBuilder().batchHint(1).build(); Map<RangeRequest, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>> ranges = keyValueService.getFirstBatchForRanges(TEST_TABLE, Iterables.limit(Iterables.cycle(rangeRequest), 100), 1); assertEquals(1, ranges.keySet().size()); assertEquals(1, ranges.values().iterator().next().getResults().size()); assertEquals("row99", PtBytes.toString(ranges.values().iterator().next().getResults().iterator().next().getRowName())); }
@Override protected Map<String, StringValue> getRangeSecondColumn(Transaction transaction, String startRowKey, String endRowKey) { SchemaApiTestTable table = tableFactory.getSchemaApiTestTable(transaction); ColumnSelection secondColSelection = SchemaApiTestTable.getColumnSelection( SchemaApiTestTable.SchemaApiTestNamedColumn.COLUMN2); RangeRequest rangeRequest = RangeRequest.builder() .startRowInclusive(SchemaApiTestRow.of(startRowKey).persistToBytes()) .endRowExclusive(SchemaApiTestRow.of(endRowKey).persistToBytes()) .retainColumns(secondColSelection) .build(); BatchingVisitableView<SchemaApiTestRowResult> rangeRequestResult = table.getRange(rangeRequest); return rangeRequestResult.immutableCopy() .stream() .collect(Collectors.toMap( entry -> entry.getRowName().getComponent1(), SchemaApiTestTable.SchemaApiTestRowResult::getColumn2)); }