@Override public Iterator<RowResult<Value>> getRange(TableReference tableRef, RangeRequest rangeRequest, long timestamp) { boolean haveOverflow = checkIfTableHasOverflowUsingNewConnection(tableRef); return Iterators.concat(new PageIterator( rangeRequest.getStartInclusive(), rangeRequest.getEndExclusive(), rangeRequest.getColumnNames(), rangeRequest.isReverse(), tableRef, haveOverflow, RangeHelpers.getMaxRowsPerPage(rangeRequest), timestamp)); }
@VisibleForTesting Set<LockEntry> allLockEntries() { Set<RowResult<Value>> results = ImmutableSet.copyOf(keyValueService.getRange( AtlasDbConstants.PERSISTED_LOCKS_TABLE, RangeRequest.all(), AtlasDbConstants.TRANSACTION_TS + 1)); return results.stream().map(LockEntry::fromRowResult).collect(Collectors.toSet()); }
public BatchingVisitableView<KeyValueRowResult> getRange(RangeRequest range) { if (range.getColumnNames().isEmpty()) { range = range.getBuilder().retainColumns(allColumns).build(); } return BatchingVisitables.transform(t.getRange(tableRef, range), new Function<RowResult<byte[]>, KeyValueRowResult>() { @Override public KeyValueRowResult apply(RowResult<byte[]> input) { return KeyValueRowResult.of(input); } }); }
/** * This is a replacement for startRow when doing reverse range request. */ public static byte[] startRowInclusiveOrLargestRow(RangeRequest rangeRequest) { Preconditions.checkArgument(rangeRequest.isReverse()); if (rangeRequest.getStartInclusive().length == 0) { return getLastRowName(); } return rangeRequest.getStartInclusive(); }
/** * This is a replacement for endRow when doing a non-reverse range request. */ public static byte[] endRowExclusiveOrOneAfterMax(RangeRequest rangeRequest) { Preconditions.checkArgument(!rangeRequest.isReverse()); if (rangeRequest.getEndExclusive().length == 0) { return oneAfterMaximumName(); } return rangeRequest.getEndExclusive(); }
@Test public void testKeyValueRanges() { putDirect("row1", "col1", "", 0); putDirect("row2", "col1", "", 0); putDirect("row2", "col2", "", 0); Map<RangeRequest, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>> ranges = keyValueService.getFirstBatchForRanges(TEST_TABLE, ImmutableList.of(RangeRequest.builder().build(), RangeRequest.builder().build()), 1); assertTrue(ranges.size() >= 1); }
@Override public FullQuery getRangeQuery(RangeRequest range, long ts, int maxRows) { List<String> bounds = Lists.newArrayListWithCapacity(2); List<Object> args = Lists.newArrayListWithCapacity(2); byte[] start = range.getStartInclusive(); byte[] end = range.getEndExclusive(); if (start.length > 0) { bounds.add(range.isReverse() ? "m.row_name <= ?" : "m.row_name >= ?"); args.add(start); } if (end.length > 0) { bounds.add(range.isReverse() ? "m.row_name > ?" : "m.row_name < ?"); args.add(end); } String query = " /* GET_RANGE_ROWS (" + tableName + ") */ " + " SELECT DISTINCT m.row_name " + " FROM " + prefixedTableName() + " m " + (bounds.isEmpty() ? "" : " WHERE " + Joiner.on(" AND ").join(bounds)) + " ORDER BY m.row_name " + (range.isReverse() ? "DESC" : "ASC") + " LIMIT " + maxRows; return new FullQuery(query).withArgs(args); }
final long timestamp) { return run((Function<DSLContext, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>>) ctx -> { int maxRows = rangeRequest.getBatchHint() == null ? 100 : (int) (1.1 * rangeRequest.getBatchHint()); Select<Record1<byte[]>> rangeQuery = getRangeQuery(ctx, tableRef, rangeRequest, timestamp, maxRows); Select<? extends Record> query; if (rangeRequest.getColumnNames().isEmpty()) { query = getLatestTimestampQueryAllColumnsSubQuery(ctx, tableRef, rangeQuery, timestamp); } else { query = getLatestTimestampQuerySomeColumnsSubQuery(ctx, tableRef, rangeQuery, rangeRequest.getColumnNames(), timestamp); if (rangeRequest.isReverse()) { valuesByRow = valuesByRow.descendingMap(); boolean mayHaveMoreResults = false; byte[] lastRow = Iterables.getLast(finalResults).getRowName(); if (!RangeRequests.isTerminalRow(rangeRequest.isReverse(), lastRow)) { nextRow = RangeRequests.getNextStartRow(rangeRequest.isReverse(), lastRow); mayHaveMoreResults = finalResults.size() == maxRows;
@Override public void deleteRange(final TableReference tableRef, final RangeRequest range) { if (range.equals(RangeRequest.all())) { try { cassandraTableTruncator.truncateTables(ImmutableSet.of(tableRef)); } catch (AtlasDbDependencyException e) { log.info("Tried to make a deleteRange({}, RangeRequest.all())" + " into a more garbage-cleanup friendly truncate(), but this failed.", LoggingArgs.tableRef(tableRef), e); super.deleteRange(tableRef, range); } } else if (isForSingleRow(range.getStartInclusive(), range.getEndExclusive())) { try { long timestamp = mutationTimestampProvider.getRemoveTimestamp(); byte[] row = range.getStartInclusive(); clientPool.runWithRetry(client -> { client.remove("deleteRange", tableRef, row, timestamp, DELETE_CONSISTENCY); return null; }); } catch (UnavailableException e) { throw new InsufficientConsistencyException( "Deleting requires all Cassandra nodes to be up and available.", e); } catch (TException e) { throw Throwables.unwrapAndThrowAtlasDbDependencyException(e); } } else { super.deleteRange(tableRef, range); } }
private NavigableMap<Cell, byte[]> getReadsInRange(TableReference table, RangeRequest range) { NavigableMap<Cell, byte[]> reads = getReadsForTable(table); if (range.getStartInclusive().length != 0) { reads = reads.tailMap(Cells.createSmallestCellForRow(range.getStartInclusive()), true); } if (range.getEndExclusive().length != 0) { reads = reads.headMap(Cells.createSmallestCellForRow(range.getEndExclusive()), false); } Map<Cell, byte[]> writes = writesByTable.get(table); if (writes != null) { reads = Maps.filterKeys(reads, Predicates.not(Predicates.in(writes.keySet()))); } if (!range.getColumnNames().isEmpty()) { Predicate<Cell> columnInNames = Predicates.compose( Predicates.in(range.getColumnNames()), Cell::getColumnName); reads = Maps.filterKeys(reads, columnInNames); } return reads; }
private byte[] copyOneTransactionInternal(RangeRequest range, long rangeId, Transaction readT, Transaction writeT) { final long maxBytes = TransactionConstants.WARN_LEVEL_FOR_QUEUED_BYTES / 2; byte[] start = getCheckpoint(rangeId, writeT); if (start == null) { return null; } RangeRequest.Builder builder = range.getBuilder().startRowInclusive(start); if (builder.isInvalidRange()) { return null; } RangeRequest rangeToUse = builder.build(); if (log.isTraceEnabled()) { log.trace("Copying table {} range {} from {} to {}", srcTable, rangeId, BaseEncoding.base16().lowerCase().encode(rangeToUse.getStartInclusive()), BaseEncoding.base16().lowerCase().encode(rangeToUse.getEndExclusive())); } BatchingVisitable<RowResult<byte[]>> bv = readT.getRange(srcTable, rangeToUse); Map<Cell, byte[]> writeMap = Maps.newHashMap(); byte[] lastRow = internalCopyRange(bv, maxBytes, writeMap); if (log.isTraceEnabled() && (lastRow != null)) { log.trace("Copying {} bytes for range {} on table {}", lastRow.length, rangeId, srcTable); } writeToKvs(writeMap); byte[] nextRow = getNextRowName(lastRow); checkpointer.checkpoint(srcTable.getQualifiedName(), rangeId, nextRow, writeT); return lastRow; }
if (request.getBatchHint() == null) { requestWithHint = request.withBatchHint(100); int batchSize = requestWithHint.getBatchHint(); final Iterator<RowResult<Value>> withLimit = Iterators.limit(range, batchSize); ImmutableList<RowResult<Value>> results = ImmutableList.copyOf(withLimit); if (results.size() != batchSize) { ret.put(request, SimpleTokenBackedResultsPage.create(request.getEndExclusive(), results, false)); return; if (RangeRequests.isTerminalRow(request.isReverse(), lastRowName)) { ret.put(request, SimpleTokenBackedResultsPage.create(lastRowName, results, false)); return; byte[] nextStartRow = RangeRequests.getNextStartRow(request.isReverse(), lastRowName); if (Arrays.equals(request.getEndExclusive(), nextStartRow)) { ret.put(request, SimpleTokenBackedResultsPage.create(nextStartRow, results, false)); } else {
@Test public void testPrefix() { byte[] end = RangeRequest.builder().prefixRange(new byte[] {-1}).build().getEndExclusive(); assertEquals(0, end.length); end = RangeRequest.builder().prefixRange(new byte[] {-2}).build().getEndExclusive(); assertEquals(1, end.length); assertEquals(-1, end[0]); end = RangeRequest.builder().prefixRange(new byte[] {0, -1}).build().getEndExclusive(); assertEquals(1, end.length); assertEquals(1, end[0]); end = RangeRequest.builder().prefixRange(new byte[] {0, -1, 0}).build().getEndExclusive(); assertEquals(3, end.length); assertEquals(1, end[2]); }
@Test public void testRangesTransactionColumnSelection() { Transaction t = startTransaction(); put(t, "row1", "col1", "v1"); t.commit(); RangeRequest range1 = RangeRequest.builder().batchHint(3).build(); RangeRequest range2 = range1.getBuilder().retainColumns(ColumnSelection.create(ImmutableSet.of(PtBytes.toBytes("col1")))).build(); t = startTransaction(); Iterable<BatchingVisitable<RowResult<byte[]>>> ranges = t.getRanges(TEST_TABLE, Iterables.limit(Iterables.cycle(range1, range2), 1000)); for (BatchingVisitable<RowResult<byte[]>> batchingVisitable : ranges) { final List<RowResult<byte[]>> list = BatchingVisitables.copyToList(batchingVisitable); assertEquals(1, list.size()); assertEquals(1, list.get(0).getColumns().size()); } RangeRequest range3 = range1.getBuilder().retainColumns(ColumnSelection.create(ImmutableSet.of(PtBytes.toBytes("col2")))).build(); verifyAllGetRangesImplsRangeSizes(t, range3, 0); }
public boolean inRange(byte[] position) { Preconditions.checkArgument(Cell.isNameValid(position)); final boolean afterStart; final boolean afterEnd; if (reverse) { afterStart = getStartInclusive().length == 0 || UnsignedBytes.lexicographicalComparator().compare(getStartInclusive(), position) >= 0; afterEnd = getEndExclusive().length == 0 || UnsignedBytes.lexicographicalComparator().compare(getEndExclusive(), position) < 0; } else { afterStart = getStartInclusive().length == 0 || UnsignedBytes.lexicographicalComparator().compare(getStartInclusive(), position) <= 0; afterEnd = getEndExclusive().length == 0 || UnsignedBytes.lexicographicalComparator().compare(getEndExclusive(), position) > 0; } return afterStart && afterEnd; }
public ClosableIterator<List<CandidateCellForSweeping>> getCandidateCellsForSweeping( TableReference tableRef, CandidateCellForSweepingRequest request) { RangeRequest range = RangeRequest.builder() .startRowInclusive(request.startRowInclusive()) .batchHint(request.batchSizeHint().orElse( PeekingIterator<RowResult<Value>> peekingValues = Iterators.peekingIterator(valueResults.get()); Iterator<List<RowResult<Set<Long>>>> tsBatches = Iterators.partition(tsResults.get(), range.getBatchHint()); Iterator<List<CandidateCellForSweeping>> candidates = Iterators.transform(tsBatches, tsBatch -> { List<CandidateCellForSweeping> candidateBatch = Lists.newArrayList();
@Override public ClosableIterator<RowResult<Value>> getBatch(int batchSize, @Nullable byte[] lastToken) { RangeRequest.Builder newRange = range.getBuilder(); if (lastToken != null) { newRange.startRowInclusive(RangeRequests.getNextStartRow(range.isReverse(), lastToken)); } newRange.batchHint(batchSize); return keyValueService.getRange(tableRef, newRange.build(), timestamp); }
public CassandraRangePagingIterable( RowGetter rowGetter, SlicePredicate slicePredicate, ColumnGetter columnGetter, RangeRequest rangeRequest, Supplier<ResultsExtractor<T>> resultsExtractor, long timestamp) { this.rowGetter = rowGetter; this.slicePredicate = slicePredicate; this.columnGetter = columnGetter; this.rangeRequest = rangeRequest; this.resultsExtractor = resultsExtractor; this.timestamp = timestamp; batchHint = rangeRequest.getBatchHint() == null ? 100 : rangeRequest.getBatchHint(); selection = rangeRequest.getColumnNames().isEmpty() ? ColumnSelection.all() : ColumnSelection.create(rangeRequest.getColumnNames()); }
public static ColumnSelection extractColumnSelection(RangeRequest rangeRequest) { if (rangeRequest.getColumnNames().isEmpty()) { return ColumnSelection.all(); } return ColumnSelection.create(rangeRequest.getColumnNames()); }
private TokenBackedBasicResultsPage<RowResult<T>, byte[]> getPage( Map<ByteBuffer, List<ColumnOrSuperColumn>> colsByKey) { return resultsExtractor.get().getPageFromRangeResults( colsByKey, timestamp, selection, rangeRequest.getEndExclusive()); }