@Override public FullQuery getRowsColumnRangeQuery(RowsColumnRangeBatchRequest batch, long ts) { List<FullQuery> fullQueries = new ArrayList<>(); batch.getPartialFirstRow() .ifPresent(entry -> fullQueries.add(getRowsColumnRangeSubQuery(entry.getKey(), ts, entry.getValue()))); if (!batch.getRowsToLoadFully().isEmpty()) { fullQueries.add(getRowsColumnRangeFullyLoadedRowsSubQuery(batch.getRowsToLoadFully(), ts, batch.getColumnRangeSelection())); } batch.getPartialLastRow() .ifPresent(entry -> fullQueries.add(getRowsColumnRangeSubQuery(entry.getKey(), ts, entry.getValue()))); List<String> subQueries = fullQueries.stream().map(FullQuery::getQuery).collect(Collectors.toList()); int totalArgs = fullQueries.stream().mapToInt(fullQuery -> fullQuery.getArgs().length).sum(); List<Object> args = fullQueries.stream() .flatMap(fullQuery -> Stream.of(fullQuery.getArgs())) .collect(Collectors.toCollection(() -> new ArrayList<>(totalArgs))); String query = Joiner.on(") UNION ALL (") .appendTo(new StringBuilder("("), subQueries) .append(")") .append(" ORDER BY row_name ASC, col_name ASC") .toString(); return new FullQuery(query).withArgs(args); }
public static List<byte[]> getAllRowsInOrder(RowsColumnRangeBatchRequest batch) { List<byte[]> allRows = new ArrayList<>(batch.getRowsToLoadFully().size() + 2); batch.getPartialFirstRow().ifPresent(row -> allRows.add(row.getKey())); allRows.addAll(batch.getRowsToLoadFully()); batch.getPartialLastRow().ifPresent(row -> allRows.add(row.getKey())); return allRows; }
@Value.Check protected void check() { Preconditions.checkState(getColumnRangeSelection() != null || getRowsToLoadFully().isEmpty(), "Must specify a column range selection when loading full rows."); } }
/** * Assumes that there are at least 2 requests after paritioning (i.e. that the last request in the partition is not * also the first). */ private static RowsColumnRangeBatchRequest getLastRequestInPartition( RowsColumnRangeBatchRequest originalRequest, List<byte[]> allRowsInLastPartition) { ImmutableRowsColumnRangeBatchRequest.Builder lastPartition = ImmutableRowsColumnRangeBatchRequest.builder() .columnRangeSelection(originalRequest.getColumnRangeSelection()); if (originalRequest.getPartialLastRow().isPresent()) { lastPartition.partialLastRow(originalRequest.getPartialLastRow()) .rowsToLoadFully(allRowsInLastPartition.subList(0, allRowsInLastPartition.size() - 1)); } else { lastPartition.rowsToLoadFully(allRowsInLastPartition); } return lastPartition.build(); } }
/** * Assumes that there are at least 2 requests after paritioning (i.e. that the first request in the partition is not * also the last). */ private static RowsColumnRangeBatchRequest getFirstRequestInPartition( RowsColumnRangeBatchRequest originalRequest, List<byte[]> allRowsInFirstPartition) { ImmutableRowsColumnRangeBatchRequest.Builder firstPartition = ImmutableRowsColumnRangeBatchRequest.builder() .columnRangeSelection(originalRequest.getColumnRangeSelection()); if (originalRequest.getPartialFirstRow().isPresent()) { firstPartition.partialFirstRow(originalRequest.getPartialFirstRow()) .rowsToLoadFully(Iterables.skip(allRowsInFirstPartition, 1)); } else { firstPartition.rowsToLoadFully(allRowsInFirstPartition); } return firstPartition.build(); }
private static void assertIntermediatePartitionsHaveNoPartialRows(List<RowsColumnRangeBatchRequest> partitions) { // No partition other than the first should have a partial first row for (RowsColumnRangeBatchRequest partition : partitions.subList(1, partitions.size())) { Assert.assertFalse(partition.getPartialFirstRow().isPresent()); } // No partition other than the last should have a partial last row for (RowsColumnRangeBatchRequest partition : partitions.subList(0, partitions.size() - 1)) { Assert.assertFalse(partition.getPartialLastRow().isPresent()); } }
/** * Partitions the given {@link RowsColumnRangeBatchRequest} into multiple, preserving the ordering of rows. Each * partitioned {@link RowsColumnRangeBatchRequest} will have exactly {@code partitionSize} rows in total, except * possibly for the last one, which may have fewer rows (but not more). No row will be split across partitions. */ public static List<RowsColumnRangeBatchRequest> partition(RowsColumnRangeBatchRequest batch, int partitionSize) { if (getAllRowsInOrder(batch).size() <= partitionSize) { return ImmutableList.of(batch); } List<List<byte[]>> partitionedRows = Lists.partition(getAllRowsInOrder(batch), partitionSize); List<RowsColumnRangeBatchRequest> partitions = new ArrayList<>(partitionedRows.size()); partitions.add(getFirstRequestInPartition(batch, partitionedRows.get(0))); for (int partitionNumber = 1; partitionNumber < partitionedRows.size() - 1; partitionNumber++) { RowsColumnRangeBatchRequest partition = ImmutableRowsColumnRangeBatchRequest.builder() .columnRangeSelection(batch.getColumnRangeSelection()) .rowsToLoadFully(partitionedRows.get(partitionNumber)) .build(); partitions.add(partition); } partitions.add(getLastRequestInPartition(batch, Iterables.getLast(partitionedRows))); return partitions; }
public static List<byte[]> getAllRowsInOrder(RowsColumnRangeBatchRequest batch) { List<byte[]> allRows = new ArrayList<>(batch.getRowsToLoadFully().size() + 2); batch.getPartialFirstRow().ifPresent(row -> allRows.add(row.getKey())); allRows.addAll(batch.getRowsToLoadFully()); batch.getPartialLastRow().ifPresent(row -> allRows.add(row.getKey())); return allRows; }
@Value.Check protected void check() { Preconditions.checkState(getColumnRangeSelection() != null || getRowsToLoadFully().isEmpty(), "Must specify a column range selection when loading full rows."); } }
/** * Assumes that there are at least 2 requests after paritioning (i.e. that the last request in the partition is not * also the first). */ private static RowsColumnRangeBatchRequest getLastRequestInPartition( RowsColumnRangeBatchRequest originalRequest, List<byte[]> allRowsInLastPartition) { ImmutableRowsColumnRangeBatchRequest.Builder lastPartition = ImmutableRowsColumnRangeBatchRequest.builder() .columnRangeSelection(originalRequest.getColumnRangeSelection()); if (originalRequest.getPartialLastRow().isPresent()) { lastPartition.partialLastRow(originalRequest.getPartialLastRow()) .rowsToLoadFully(allRowsInLastPartition.subList(0, allRowsInLastPartition.size() - 1)); } else { lastPartition.rowsToLoadFully(allRowsInLastPartition); } return lastPartition.build(); } }
/** * Assumes that there are at least 2 requests after paritioning (i.e. that the first request in the partition is not * also the last). */ private static RowsColumnRangeBatchRequest getFirstRequestInPartition( RowsColumnRangeBatchRequest originalRequest, List<byte[]> allRowsInFirstPartition) { ImmutableRowsColumnRangeBatchRequest.Builder firstPartition = ImmutableRowsColumnRangeBatchRequest.builder() .columnRangeSelection(originalRequest.getColumnRangeSelection()); if (originalRequest.getPartialFirstRow().isPresent()) { firstPartition.partialFirstRow(originalRequest.getPartialFirstRow()) .rowsToLoadFully(Iterables.skip(allRowsInFirstPartition, 1)); } else { firstPartition.rowsToLoadFully(allRowsInFirstPartition); } return firstPartition.build(); }
/** * Partitions the given {@link RowsColumnRangeBatchRequest} into multiple, preserving the ordering of rows. Each * partitioned {@link RowsColumnRangeBatchRequest} will have exactly {@code partitionSize} rows in total, except * possibly for the last one, which may have fewer rows (but not more). No row will be split across partitions. */ public static List<RowsColumnRangeBatchRequest> partition(RowsColumnRangeBatchRequest batch, int partitionSize) { if (getAllRowsInOrder(batch).size() <= partitionSize) { return ImmutableList.of(batch); } List<List<byte[]>> partitionedRows = Lists.partition(getAllRowsInOrder(batch), partitionSize); List<RowsColumnRangeBatchRequest> partitions = new ArrayList<>(partitionedRows.size()); partitions.add(getFirstRequestInPartition(batch, partitionedRows.get(0))); for (int partitionNumber = 1; partitionNumber < partitionedRows.size() - 1; partitionNumber++) { RowsColumnRangeBatchRequest partition = ImmutableRowsColumnRangeBatchRequest.builder() .columnRangeSelection(batch.getColumnRangeSelection()) .rowsToLoadFully(partitionedRows.get(partitionNumber)) .build(); partitions.add(partition); } partitions.add(getLastRequestInPartition(batch, Iterables.getLast(partitionedRows))); return partitions; }
private static void assertColumnRangesInPartitionsMatchOriginal( RowsColumnRangeBatchRequest request, List<RowsColumnRangeBatchRequest> partitions) { Assert.assertEquals(request.getPartialFirstRow(), partitions.get(0).getPartialFirstRow()); Assert.assertEquals(request.getPartialLastRow(), partitions.get(partitions.size() - 1).getPartialLastRow()); for (RowsColumnRangeBatchRequest partition : partitions) { Assert.assertTrue(partition.getRowsToLoadFully().isEmpty() || partition.getColumnRangeSelection().equals(request.getColumnRangeSelection())); } }
@Override public FullQuery getRowsColumnRangeQuery(RowsColumnRangeBatchRequest batch, long ts) { List<FullQuery> fullQueries = new ArrayList<>(); batch.getPartialFirstRow() .ifPresent(entry -> fullQueries.add(getRowsColumnRangeSubQuery(entry.getKey(), ts, entry.getValue()))); if (!batch.getRowsToLoadFully().isEmpty()) { fullQueries.add(getRowsColumnRangeFullyLoadedRowsSubQuery(batch.getRowsToLoadFully(), ts, batch.getColumnRangeSelection())); } batch.getPartialLastRow() .ifPresent(entry -> fullQueries.add(getRowsColumnRangeSubQuery(entry.getKey(), ts, entry.getValue()))); List<String> subQueries = fullQueries.stream().map(FullQuery::getQuery).collect(Collectors.toList()); int totalArgs = fullQueries.stream().mapToInt(fullQuery -> fullQuery.getArgs().length).sum(); List<Object> args = fullQueries.stream() .flatMap(fullQuery -> Stream.of(fullQuery.getArgs())) .collect(Collectors.toCollection(() -> new ArrayList<>(totalArgs))); String query = Joiner.on(") UNION ALL (") .appendTo(new StringBuilder("("), subQueries) .append(")") .append(" ORDER BY row_name ASC, col_name ASC") .toString(); return new FullQuery(query).withArgs(args); }