/** * Split this query into multiple queries that logically combine into this query. This is intended * to be used by map reduce style frameworks like Beam to split a query across multiple workers. * * <p>Expected Usage: * * <pre>{@code * List<ByteString> splitPoints = ...; * List<Query> queryShards = myQuery.shard(splitPoints); * List<ApiFuture<List<Row>>> futures = new ArrayList(); * for (Query subQuery : queryShards) { * futures.add(dataClient.readRowsCallable().all().futureCall(subQuery)); * } * List<List<Row>> results = ApiFutures.allAsList(futures).get(); * }</pre> */ public List<Query> shard(SortedSet<ByteString> splitPoints) { Preconditions.checkState(builder.getRowsLimit() == 0, "Can't shard a query with a row limit"); List<RowSet> shardedRowSets = RowSetUtil.shard(builder.getRows(), splitPoints); List<Query> shards = Lists.newArrayListWithCapacity(shardedRowSets.size()); for (RowSet rowSet : shardedRowSets) { Query queryShard = new Query(tableId); queryShard.builder.mergeFrom(this.builder.build()); queryShard.builder.setRows(rowSet); shards.add(queryShard); } return shards; }
.setTableNameBytes(ByteStringer.wrap(lastTableBytes)) .setFilter(filter) .setRows(RowSet.newBuilder() .addRowKeys(ByteStringer.wrap(key.getBytes())));
.setTableNameBytes(ByteStringer.wrap(lastTableBytes)) .setFilter(filter) .setRows(rowSet);
.setTableName(tableName) .setFilter(filter) .setRows(RowSet.newBuilder().addAllRowKeys(futures.keys()).build()) .build() );
Builder builder = originalRequest.toBuilder().setRows(remaining);
/** * Split this query into multiple queries that logically combine into this query. This is intended * to be used by map reduce style frameworks like Beam to split a query across multiple workers. * * <p>Expected Usage: * * <pre>{@code * List<ByteString> splitPoints = ...; * List<Query> queryShards = myQuery.shard(splitPoints); * List<ApiFuture<List<Row>>> futures = new ArrayList(); * for (Query subQuery : queryShards) { * futures.add(dataClient.readRowsCallable().all().futureCall(subQuery)); * } * List<List<Row>> results = ApiFutures.allAsList(futures).get(); * }</pre> */ public List<Query> shard(SortedSet<ByteString> splitPoints) { Preconditions.checkState(builder.getRowsLimit() == 0, "Can't shard a query with a row limit"); List<RowSet> shardedRowSets = RowSetUtil.shard(builder.getRows(), splitPoints); List<Query> shards = Lists.newArrayListWithCapacity(shardedRowSets.size()); for (RowSet rowSet : shardedRowSets) { Query queryShard = new Query(tableId); queryShard.builder.mergeFrom(this.builder.build()); queryShard.builder.setRows(rowSet); shards.add(queryShard); } return shards; }
/** Helper function to get a table's data. */ private List<KV<ByteString, ByteString>> getTableData(String tableName) throws IOException { // Add empty range to avoid TARGET_NOT_SET error RowRange range = RowRange.newBuilder() .setStartKeyClosed(ByteString.EMPTY) .setEndKeyOpen(ByteString.EMPTY) .build(); RowSet rowSet = RowSet.newBuilder().addRowRanges(range).build(); ReadRowsRequest.Builder readRowsRequestBuilder = ReadRowsRequest.newBuilder().setTableName(tableName).setRows(rowSet); ResultScanner<Row> scanner = session.getDataClient().readRows(readRowsRequestBuilder.build()); Row currentRow; List<KV<ByteString, ByteString>> tableData = new ArrayList<>(); while ((currentRow = scanner.next()) != null) { ByteString key = currentRow.getKey(); ByteString value = currentRow.getFamilies(0).getColumns(0).getCells(0).getValue(); tableData.add(KV.of(key, value)); } scanner.close(); return tableData; }
@Override public boolean start() throws IOException { RowSet.Builder rowSetBuilder = RowSet.newBuilder(); for (ByteKeyRange sourceRange : source.getRanges()) { rowSetBuilder = rowSetBuilder.addRowRanges( RowRange.newBuilder() .setStartKeyClosed(ByteString.copyFrom(sourceRange.getStartKey().getValue())) .setEndKeyOpen(ByteString.copyFrom(sourceRange.getEndKey().getValue()))); } RowSet rowSet = rowSetBuilder.build(); String tableNameSr = session.getOptions().getInstanceName().toTableNameStr(source.getTableId().get()); ReadRowsRequest.Builder requestB = ReadRowsRequest.newBuilder().setRows(rowSet).setTableName(tableNameSr); if (source.getRowFilter() != null) { requestB.setFilter(source.getRowFilter()); } results = session.getDataClient().readRows(requestB.build()); return advance(); }
ReadRowsRequest buildUpdatedRequest() { ReadRowsRequest.Builder newRequest = ReadRowsRequest.newBuilder() .setRows(filterRows()) .setTableName(originalRequest.getTableName()) .setAppProfileId(originalRequest.getAppProfileId()); if (originalRequest.hasFilter()) { newRequest.setFilter(originalRequest.getFilter()); } // If the row limit is set, update it. long numRowsLimit = originalRequest.getRowsLimit(); if (numRowsLimit > 0) { // Updates the {@code numRowsLimit} by removing the number of rows already read. numRowsLimit -= rowCount; checkArgument(numRowsLimit > 0, "The remaining number of rows must be greater than 0."); newRequest.setRowsLimit(numRowsLimit); } return newRequest.build(); }
/** {@inheritDoc} */ @Override public ReadRowsRequest.Builder adapt(Get operation, ReadHooks readHooks) { Scan operationAsScan = new Scan(addKeyOnlyFilter(operation)); scanAdapter.throwIfUnsupportedScan(operationAsScan); return ReadRowsRequest.newBuilder() .setFilter(scanAdapter.buildFilter(operationAsScan, readHooks).toProto()) .setRows(RowSet.newBuilder().addRowKeys(ByteString.copyFrom(operation.getRow()))); }
/** {@inheritDoc} */ @Override public ReadRowsRequest.Builder adapt(Scan scan, ReadHooks readHooks) { throwIfUnsupportedScan(scan); ReadRowsRequest.Builder requestBuilder = ReadRowsRequest.newBuilder() .setRows(toRowSet(scan)) .setFilter(buildFilter(scan, readHooks).toProto()); if (LIMIT_AVAILABLE && scan.getLimit() > 0) { requestBuilder.setRowsLimit(scan.getLimit()); } return requestBuilder; }
.setTableName(NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, TABLE_ID)) .setAppProfileId(APP_PROFILE_ID) .setRows( RowSet.newBuilder() .addRowRanges( .setTableName(NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, TABLE_ID)) .setAppProfileId(APP_PROFILE_ID) .setRows( RowSet.newBuilder() .addRowRanges(
Builder builder = originalRequest.toBuilder().setRows(remaining);
.setTableName(NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, TABLE_ID)) .setAppProfileId(APP_PROFILE_ID) .setRows( RowSet.newBuilder() .addRowRanges( .setTableName(NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, TABLE_ID)) .setAppProfileId(APP_PROFILE_ID) .setRows( RowSet.newBuilder() .addRowRanges(
@Test public void testFromProto() { ReadRowsRequest request = ReadRowsRequest.newBuilder() .setTableName(NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, TABLE_ID)) .setAppProfileId(APP_PROFILE_ID) .setFilter(RowFilter.newBuilder().setRowKeyRegexFilter(ByteString.copyFromUtf8(".*"))) .setRows( RowSet.newBuilder() .addRowKeys(ByteString.copyFromUtf8("row-key")) .addRowRanges( RowRange.newBuilder() .setStartKeyClosed(ByteString.copyFromUtf8("j")) .setEndKeyClosed(ByteString.copyFromUtf8("z")))) .build(); Query query = Query.fromProto(request); assertThat(query.toProto(requestContext)).isEqualTo(request); }