public CloseableIterator<T> results() { final List<RangeReadInfo> reads = new ArrayList<>(); for (final SinglePartitionQueryRanges r : ranges) { for (final ByteArrayRange range : r.getSortKeyRanges()) { reads.add(new RangeReadInfo(r.getPartitionKey(), range)); } } return executeQuery(reads); }
public QueryRanges(final byte[][] partitionKeys, final QueryRanges queryRanges) { if ((queryRanges == null) || (queryRanges.partitionRanges == null) || queryRanges.partitionRanges.isEmpty()) { partitionRanges = fromPartitionKeys(partitionKeys); } else if ((partitionKeys == null) || (partitionKeys.length == 0)) { partitionRanges = queryRanges.partitionRanges; } else { partitionRanges = new ArrayList<>(partitionKeys.length * queryRanges.partitionRanges.size()); for (final byte[] partitionKey : partitionKeys) { for (final SinglePartitionQueryRanges sortKeyRange : queryRanges.partitionRanges) { byte[] newPartitionKey; if (partitionKey == null) { newPartitionKey = sortKeyRange.getPartitionKey(); } else if (sortKeyRange.getPartitionKey() == null) { newPartitionKey = partitionKey; } else { newPartitionKey = ByteArrayUtils.combineArrays(partitionKey, sortKeyRange.getPartitionKey()); } partitionRanges.add( new SinglePartitionQueryRanges(newPartitionKey, sortKeyRange.getSortKeyRanges())); } } } }
public QueryRanges(final ByteArrayRange singleSortKeyRange) { partitionRanges = Collections.singletonList(new SinglePartitionQueryRanges(singleSortKeyRange)); }
public boolean isMultiRange() { if (compositeQueryRanges != null) { return compositeQueryRanges.size() >= 2; } if (partitionRanges.isEmpty()) { return false; } if (partitionRanges.size() > 1) { return true; } final SinglePartitionQueryRanges partition = partitionRanges.iterator().next(); if ((partition.getSortKeyRanges() != null) && (partition.getSortKeyRanges().size() <= 1)) { return false; } return true; } }
public static byte[][] getQueryPartitionKeys( final NumericIndexStrategy strategy, final MultiDimensionalNumericData queryData, final IndexMetaData... hints) { final QueryRanges queryRanges = strategy.getQueryRanges(queryData, hints); return queryRanges.getPartitionQueryRanges().stream().map( input -> input.getPartitionKey()).toArray(i -> new byte[i][]); }
private static Collection<SinglePartitionQueryRanges> fromPartitionKeys( final byte[][] partitionKeys) { if (partitionKeys == null) { return null; } return Arrays.stream(partitionKeys).map(input -> new SinglePartitionQueryRanges(input)).collect( Collectors.toList()); }
private List<QueryRequest> addQueryRanges( final String tableName, final SinglePartitionQueryRanges r, short[] adapterIds, final InternalAdapterStore adapterStore) { final List<QueryRequest> retVal = new ArrayList<>(); final byte[] partitionKey = ((r.getPartitionKey() == null) || (r.getPartitionKey().length == 0)) ? DynamoDBWriter.EMPTY_PARTITION_KEY : r.getPartitionKey(); if (((adapterIds == null) || (adapterIds.length == 0)) && (adapterStore != null)) { adapterIds = adapterStore.getAdapterIds(); } for (final Short adapterId : adapterIds) { final Collection<ByteArrayRange> sortKeyRanges = r.getSortKeyRanges(); if ((sortKeyRanges != null) && !sortKeyRanges.isEmpty()) { sortKeyRanges.forEach( (sortKeyRange -> retVal.add( getQuery(tableName, partitionKey, sortKeyRange, adapterId)))); } else { retVal.add(getQuery(tableName, partitionKey, null, adapterId)); } } return retVal; }
public QueryRanges(final List<QueryRanges> queryRangesList) { // group by partition final Map<ByteArray, Collection<ByteArrayRange>> sortRangesPerPartition = new HashMap<>(); for (final QueryRanges qr : queryRangesList) { for (final SinglePartitionQueryRanges r : qr.getPartitionQueryRanges()) { final Collection<ByteArrayRange> ranges = sortRangesPerPartition.get(new ByteArray(r.getPartitionKey())); if (ranges == null) { sortRangesPerPartition.put( new ByteArray(r.getPartitionKey()), new ArrayList<>(r.getSortKeyRanges())); } else { ranges.addAll(r.getSortKeyRanges()); } } } partitionRanges = new ArrayList<>(sortRangesPerPartition.size()); for (final Entry<ByteArray, Collection<ByteArrayRange>> e : sortRangesPerPartition.entrySet()) { Collection<ByteArrayRange> mergedRanges; if (e.getValue() != null) { mergedRanges = ByteArrayRange.mergeIntersections(e.getValue(), MergeOperation.UNION); } else { mergedRanges = null; } partitionRanges.add(new SinglePartitionQueryRanges(e.getKey().getBytes(), mergedRanges)); } }
@Override public SinglePartitionQueryRanges apply(final SinglePartitionInsertionIds input) { return new SinglePartitionQueryRanges( input.getPartitionKey(), Collections2.transform( input.getSortKeys(), new Function<byte[], ByteArrayRange>() { @Override public ByteArrayRange apply(final byte[] input) { return new ByteArrayRange(input, input, false); } })); } }));
if ((partition.getSortKeyRanges() == null) || partition.getSortKeyRanges().isEmpty()) { internalQueryRanges.add( new ByteArrayRange(partition.getPartitionKey(), partition.getPartitionKey())); } else if (partition.getPartitionKey() == null) { internalQueryRanges.addAll(partition.getSortKeyRanges()); } else { for (final ByteArrayRange sortKeyRange : partition.getSortKeyRanges()) { internalQueryRanges.add( new ByteArrayRange( ByteArrayUtils.combineArrays( partition.getPartitionKey(), sortKeyRange.getStart()), ByteArrayUtils.combineArrays(partition.getPartitionKey(), sortKeyRange.getEnd()), sortKeyRange.singleValue));
public BaseRowPrefixQuery( final Index index, final byte[] partitionKey, final byte[] sortKeyPrefix, final ScanCallback<T, ?> scanCallback, final DifferingFieldVisibilityEntryCount differingVisibilityCounts, final FieldVisibilityCount visibilityCounts, final DataIndexRetrieval dataIndexRetrieval, final String[] authorizations) { super( index, authorizations, scanCallback, differingVisibilityCounts, visibilityCounts, dataIndexRetrieval); final ByteArrayRange sortKeyPrefixRange = new ByteArrayRange(sortKeyPrefix, sortKeyPrefix, false); final List<SinglePartitionQueryRanges> ranges = new ArrayList<>(); final Collection<ByteArrayRange> sortKeys = Collections.singleton(sortKeyPrefixRange); ranges.add(new SinglePartitionQueryRanges(partitionKey, sortKeys)); queryRanges = new QueryRanges(ranges); }
public CloseableIterator<T> results() { final List<RangeReadInfo> reads = new ArrayList<>(); for (final SinglePartitionQueryRanges r : ranges) { for (final ByteArrayRange range : r.getSortKeyRanges()) { final double start = range.getStart() != null ? RedisUtils.getScore(range.getStart()) : Double.NEGATIVE_INFINITY; final double end = range.getEnd() != null ? RedisUtils.getScore(range.getEndAsNextPrefix()) : Double.POSITIVE_INFINITY; reads.add(new RangeReadInfo(r.getPartitionKey(), start, end)); } } if (async) { return executeQueryAsync(reads); } else { return executeQuery(reads); } }
public static List<SinglePartitionQueryRanges> getQueryRanges( final List<BinnedNumericDataset> binnedQueries, final SpaceFillingCurve sfc, final int maxRanges, final byte tier) { final List<SinglePartitionQueryRanges> queryRanges = new ArrayList<>(); int maxRangeDecompositionPerBin = maxRanges; if ((maxRanges > 1) && (binnedQueries.size() > 1)) { maxRangeDecompositionPerBin = (int) Math.ceil((double) maxRanges / (double) binnedQueries.size()); } for (final BinnedNumericDataset binnedQuery : binnedQueries) { final RangeDecomposition rangeDecomp = sfc.decomposeRange(binnedQuery, true, maxRangeDecompositionPerBin); final byte[] tierAndBinId = ByteArrayUtils.combineArrays(new byte[] {tier // we're assuming tiers only go to 127 (the max byte // value) }, binnedQuery.getBinId()); queryRanges.add( new SinglePartitionQueryRanges(tierAndBinId, Arrays.asList(rangeDecomp.getRanges()))); } return queryRanges; }
for (final SinglePartitionQueryRanges r : ranges) { final byte[] partitionKey = ((r.getPartitionKey() == null) || (r.getPartitionKey().length == 0)) ? CassandraUtils.EMPTY_PARTITION_KEY : r.getPartitionKey(); for (final ByteArrayRange range : r.getSortKeyRanges()) { final BoundStatement boundRead = new BoundStatement(preparedRead); final byte[] start = range.getStart() != null ? range.getStart() : new byte[0];
/** * Always returns a single range since this is a 1-dimensional index. The sort-order of the bytes * is the same as the sort order of values, so an indexedRange can be represented by a single * contiguous ByteArrayRange. {@inheritDoc} */ @Override public QueryRanges getQueryRanges( final MultiDimensionalNumericData indexedRange, final int maxEstimatedRangeDecomposition, final IndexMetaData... hints) { final T min = cast(indexedRange.getMinValuesPerDimension()[0]); final byte[] start = lexicoder.toByteArray(min); final T max = cast(Math.ceil(indexedRange.getMaxValuesPerDimension()[0])); final byte[] end = lexicoder.toByteArray(max); final ByteArrayRange range = new ByteArrayRange(start, end); final SinglePartitionQueryRanges partitionRange = new SinglePartitionQueryRanges(Collections.singletonList(range)); return new QueryRanges(Collections.singletonList(partitionRange)); }
for (final ByteArrayRange r : p.getSortKeyRanges()) { final SortedSet<MemoryStoreEntry> set; if (r.isSingleValue()) { set = internalData.subSet( new MemoryStoreEntry(p.getPartitionKey(), r.getStart()), new MemoryStoreEntry( p.getPartitionKey(), ByteArrayUtils.getNextPrefix(r.getStart()))); } else { set = internalData.tailSet( new MemoryStoreEntry(p.getPartitionKey(), r.getStart())).headSet( new MemoryStoreEntry(p.getPartitionKey(), r.getEndAsNextPrefix()));
private CloseableIterator<T> createIteratorForRecordReader( final RocksDBClient client, final RecordReaderParams recordReaderParams) { final GeoWaveRowRange range = recordReaderParams.getRowRange(); final byte[] startKey = range.isInfiniteStartSortKey() ? null : range.getStartSortKey(); final byte[] stopKey = range.isInfiniteStopSortKey() ? null : range.getEndSortKey(); final SinglePartitionQueryRanges partitionRange = new SinglePartitionQueryRanges( range.getPartitionKey(), Collections.singleton(new ByteArrayRange(startKey, stopKey))); final Set<String> authorizations = Sets.newHashSet(recordReaderParams.getAdditionalAuthorizations()); return createIterator( client, (RangeReaderParams<T>) recordReaderParams, (GeoWaveRowIteratorTransformer<T>) GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER, Collections.singleton(partitionRange), authorizations, // there should already be sufficient parallelism created by // input splits for record reader use cases false); }
public static <T> long cardinality( final Index index, final Map<StatisticsId, InternalDataStatistics<T, ?, ?>> stats, final QueryRanges queryRanges) { long count = 0; for (final SinglePartitionQueryRanges partitionRange : queryRanges.getPartitionQueryRanges()) { final RowRangeHistogramStatistics rangeStats = (RowRangeHistogramStatistics) stats.get( StatisticsQueryBuilder.newBuilder().factory().rowHistogram().indexName( index.getName()).partition(partitionRange.getPartitionKey()).build().getId()); if (rangeStats == null) { return Long.MAX_VALUE - 1; } for (final ByteArrayRange range : partitionRange.getSortKeyRanges()) { count += rangeStats.cardinality(range.getStart(), range.getEnd()); } } return count; }
private CloseableIterator<T> createIteratorForRecordReader( final RedissonClient client, final Compression compression, final RecordReaderParams recordReaderParams, final String namespace, final boolean visibilityEnabled) { final GeoWaveRowRange range = recordReaderParams.getRowRange(); final byte[] startKey = range.isInfiniteStartSortKey() ? null : range.getStartSortKey(); final byte[] stopKey = range.isInfiniteStopSortKey() ? null : range.getEndSortKey(); final SinglePartitionQueryRanges partitionRange = new SinglePartitionQueryRanges( range.getPartitionKey(), Collections.singleton(new ByteArrayRange(startKey, stopKey))); final Set<String> authorizations = Sets.newHashSet(recordReaderParams.getAdditionalAuthorizations()); return createIterator( client, compression, (RangeReaderParams<T>) recordReaderParams, (GeoWaveRowIteratorTransformer<T>) GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER, namespace, Collections.singleton(partitionRange), authorizations, visibilityEnabled, // there should already be sufficient parallelism created by // input splits for record reader use cases false); }
protected void initRecordScanner() { final short[] adapterIds = recordReaderParams.getAdapterIds() != null ? recordReaderParams.getAdapterIds() : new short[0]; final GeoWaveRowRange range = recordReaderParams.getRowRange(); final byte[] startKey = range.isInfiniteStartSortKey() ? null : range.getStartSortKey(); final byte[] stopKey = range.isInfiniteStopSortKey() ? null : range.getEndSortKey(); final SinglePartitionQueryRanges partitionRange = new SinglePartitionQueryRanges( range.getPartitionKey(), Collections.singleton(new ByteArrayRange(startKey, stopKey))); final Set<String> authorizations = Sets.newHashSet(recordReaderParams.getAdditionalAuthorizations()); iterator = operations.getBatchedRangeRead( recordReaderParams.getIndex().getName(), adapterIds, Collections.singleton(partitionRange), DataStoreUtils.isMergingIteratorRequired(recordReaderParams, visibilityEnabled), rowTransformer, new ClientVisibilityFilter(authorizations)).results(); }