default <T> Deleter<T> createDeleter(final ReaderParams<T> readerParams) { return new QueryAndDeleteByRow<>( createRowDeleter( readerParams.getIndex().getName(), readerParams.getAdapterStore(), readerParams.getInternalAdapterStore(), readerParams.getAdditionalAuthorizations()), createReader(readerParams)); }
protected void initScanner() { final String tableName = operations.getQualifiedTableName(readerParams.getIndex().getName()); readerParams.getQueryRanges().getPartitionQueryRanges(); tableName, queryRequest, readerParams.getAdapterIds(), readerParams.getInternalAdapterStore())))); tableName, DataStoreUtils.isMergingIteratorRequired(readerParams, visibilityEnabled), readerParams.getMaxResolutionSubsamplingPerDimension() == null);
public HBaseReader(final ReaderParams<T> readerParams, final HBaseOperations operations) { this.readerParams = readerParams; this.recordReaderParams = null; this.operations = operations; this.partitionKeyLength = readerParams.getIndex().getIndexStrategy().getPartitionKeyLength(); this.wholeRowEncoding = readerParams.isMixedVisibility() && !readerParams.isServersideAggregation(); this.clientSideRowMerging = readerParams.isClientsideRowMerging(); this.rowTransformer = readerParams.getRowTransformer(); this.scanProvider = createScanProvider(readerParams, operations, this.clientSideRowMerging); if (readerParams.isServersideAggregation()) { this.scanner = null; scanIt = (Iterator) operations.aggregateServerSide(readerParams); } else { initScanner(); } }
public DynamoDBReader( final ReaderParams<T> readerParams, final DynamoDBOperations operations, final boolean visibilityEnabled) { this.readerParams = readerParams; recordReaderParams = null; processAuthorizations(readerParams.getAdditionalAuthorizations(), readerParams); this.operations = operations; this.rowTransformer = readerParams.getRowTransformer(); this.visibilityEnabled = visibilityEnabled; initScanner(); }
final boolean async) { final Collection<SinglePartitionQueryRanges> ranges = readerParams.getQueryRanges().getPartitionQueryRanges(); final Set<String> authorizations = Sets.newHashSet(readerParams.getAdditionalAuthorizations()); if ((ranges != null) && !ranges.isEmpty()) { return createIterator( client, readerParams, readerParams.getRowTransformer(), ranges, authorizations, } else { final List<CloseableIterator<GeoWaveRow>> iterators = new ArrayList<>(); for (final short adapterId : readerParams.getAdapterIds()) { final Pair<Boolean, Boolean> groupByRowAndSortByTime = RocksDBUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId); final String indexNamePrefix = RocksDBUtils.getTablePrefix( readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName()); final Stream<CloseableIterator<GeoWaveRow>> streamIt = RocksDBUtils.getPartitions(client.getSubDirectory(), indexNamePrefix).stream().map(
@Override public <T> RowReader<T> createReader(final ReaderParams<T> readerParams) { final SortedSet<MemoryStoreEntry> internalData = storeData.get(readerParams.getIndex().getName()); int counter = 0; List<MemoryStoreEntry> retVal = new ArrayList<>(); final Collection<SinglePartitionQueryRanges> partitionRanges = readerParams.getQueryRanges().getPartitionQueryRanges(); if ((partitionRanges == null) || partitionRanges.isEmpty()) { retVal.addAll(internalData); if (!isAuthorized(it.next(), readerParams.getAdditionalAuthorizations())) { it.remove(); if ((readerParams.getLimit() != null) && (readerParams.getLimit() > 0) && (retVal.size() > readerParams.getLimit())) { retVal = retVal.subList(0, readerParams.getLimit()); if (!isAuthorized(it.next(), readerParams.getAdditionalAuthorizations())) { it.remove(); if ((readerParams.getLimit() != null) && (readerParams.getLimit() > 0) && ((counter + set.size()) > readerParams.getLimit())) { final List<MemoryStoreEntry> subset = new ArrayList<>(set); retVal.addAll(subset.subList(0, readerParams.getLimit() - counter)); break; } else { retVal.addAll(set);
protected void initScanner() { final Collection<SinglePartitionQueryRanges> ranges = readerParams.getQueryRanges().getPartitionQueryRanges(); if ((ranges != null) && !ranges.isEmpty()) { iterator = operations.getBatchedRangeRead( readerParams.getIndex().getName(), readerParams.getAdapterIds(), ranges, DataStoreUtils.isMergingIteratorRequired(readerParams, visibilityEnabled), rowTransformer, new ClientVisibilityFilter( Sets.newHashSet(readerParams.getAdditionalAuthorizations()))).results(); } else { final Select select = operations.getSelect(readerParams.getIndex().getName()); CloseableIterator<CassandraRow> results = operations.executeQuery(select); if ((readerParams.getAdapterIds() != null) && (readerParams.getAdapterIds().length > 0)) {
public <T> Iterator<GeoWaveRow> aggregateServerSide(final ReaderParams<T> readerParams) { final String tableName = readerParams.getIndex().getName(); final Aggregation aggregation = readerParams.getAggregation().getRight(); AggregationProtosClient.AggregationRequest.newBuilder(); requestBuilder.setAggregation(aggregationBuilder.build()); if (readerParams.getFilter() != null) { final List<QueryFilter> distFilters = new ArrayList(); distFilters.add(readerParams.getFilter()); } else { final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges(); if (!coords.isEmpty()) { final byte[] filterBytes = new HBaseNumericIndexStrategyFilter( readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray(); final ByteString filterByteString = URLClassloaderUtils.toBinary(readerParams.getIndex().getIndexModel()))); readerParams.getMaxRangeDecomposition() == null ? options.getAggregationMaxRangeDecomposition() : readerParams.getMaxRangeDecomposition(); final MultiRowRangeFilter multiFilter = getMultiRowRangeFilter( DataStoreUtils.constraintsToQueryRanges( readerParams.getConstraints(),
public void bulkDelete(final ReaderParams readerParams) { final String tableName = readerParams.getIndex().getName(); final short[] adapterIds = readerParams.getAdapterIds(); Long total = 0L; if (readerParams.getFilter() != null) { final List<QueryFilter> distFilters = new ArrayList(); distFilters.add(readerParams.getFilter()); } else { final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges(); if (!coords.isEmpty()) { final byte[] filterBytes = new HBaseNumericIndexStrategyFilter( readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray(); final ByteString filterByteString = ByteString.copyFrom(PersistenceUtils.toBinary(readerParams.getIndex().getIndexModel()))); getMultiRowRangeFilter(readerParams.getQueryRanges().getCompositeQueryRanges()); if (multiFilter != null) { requestBuilder.setRangeFilter(ByteString.copyFrom(multiFilter.toByteArray())); final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges(); if ((ranges != null) && !ranges.isEmpty()) { final ByteArrayRange aggRange = ByteArrayUtils.getSingleRange(ranges);
private void addDistFilter(final ReaderParams<T> params, final FilterList filterList) { final HBaseDistributableFilter hbdFilter = new HBaseDistributableFilter(); if (wholeRowEncoding) { hbdFilter.setWholeRowFilter(true); } hbdFilter.setPartitionKeyLength(partitionKeyLength); final List<QueryFilter> distFilters = Lists.newArrayList(); distFilters.add(params.getFilter()); hbdFilter.init( distFilters, params.getIndex().getIndexModel(), params.getAdditionalAuthorizations()); filterList.addFilter(hbdFilter); }
if (readerParams.getFilter() != null) { addDistFilter(readerParams, filterList); } else { scanProvider, operations, readerParams.getQueryRanges().getCompositeQueryRanges(), partitionKeyLength); operations.startParallelScan(parallelScanner, readerParams.getIndex().getName()); scanner = parallelScanner; } catch (final Exception e) { try { final Iterable<Result> iterable = operations.getScannedResults(multiScanner, readerParams.getIndex().getName()); if (iterable instanceof ResultScanner) { this.scanner = (ResultScanner) iterable;
public CassandraReader( final ReaderParams<T> readerParams, final CassandraOperations operations, final boolean visibilityEnabled) { this.readerParams = readerParams; recordReaderParams = null; this.operations = operations; this.rowTransformer = readerParams.getRowTransformer(); this.visibilityEnabled = visibilityEnabled; initScanner(); }
private void addIndexFilter(final ReaderParams<T> params, final FilterList filterList) { final List<MultiDimensionalCoordinateRangesArray> coords = params.getCoordinateRanges(); if ((coords != null) && !coords.isEmpty()) { final HBaseNumericIndexStrategyFilter numericIndexFilter = new HBaseNumericIndexStrategyFilter( params.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})); filterList.addFilter(numericIndexFilter); } }
@Override public boolean apply(final MemoryStoreEntry input) { if ((readerParams.getFilter() != null) && options.isServerSideLibraryEnabled()) { final PersistentDataset<CommonIndexValue> commonData = new PersistentDataset<>(); final List<FlattenedUnreadData> unreadData = new ArrayList<>(); final List<String> commonIndexFieldNames = DataStoreUtils.getUniqueDimensionFields(readerParams.getIndex().getIndexModel()); for (final GeoWaveValue v : input.getRow().getFieldValues()) { unreadData.add( DataStoreUtils.aggregateFieldData( input.getRow(), v, commonData, readerParams.getIndex().getIndexModel(), commonIndexFieldNames)); } return readerParams.getFilter().accept( readerParams.getIndex().getIndexModel(), new DeferredReadCommonIndexedPersistenceEncoding( input.getRow().getAdapterId(), input.getRow().getDataId(), input.getRow().getPartitionKey(), input.getRow().getSortKey(), input.getRow().getNumberOfDuplicates(), commonData, unreadData.isEmpty() ? null : new UnreadFieldDataList(unreadData))); } return true; } }), readerParams.getRowTransformer());
@Override public boolean apply(final CassandraRow input) { return Arrays.contains(readerParams.getAdapterIds(), input.getAdapterId()); } }));
protected Scan getMultiScanner(final FilterList filterList) { // Single scan w/ multiple ranges final Scan multiScanner = scanProvider.get(); final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges(); final MultiRowRangeFilter filter = operations.getMultiRowRangeFilter(ranges); if (filter != null) { filterList.addFilter(filter); final List<RowRange> rowRanges = filter.getRowRanges(); multiScanner.setStartRow(rowRanges.get(0).getStartRow()); final RowRange stopRowRange = rowRanges.get(rowRanges.size() - 1); byte[] stopRowExclusive; if (stopRowRange.isStopRowInclusive()) { // because the end is always exclusive, to make an inclusive // stop row into exlusive all we need to do is add a traling 0 stopRowExclusive = HBaseUtils.getInclusiveEndKey(stopRowRange.getStopRow()); } else { stopRowExclusive = stopRowRange.getStopRow(); } multiScanner.setStopRow(stopRowExclusive); } return multiScanner; }
final boolean async) { final Collection<SinglePartitionQueryRanges> ranges = readerParams.getQueryRanges().getPartitionQueryRanges(); final Set<String> authorizations = Sets.newHashSet(readerParams.getAdditionalAuthorizations()); if ((ranges != null) && !ranges.isEmpty()) { return createIterator( compression, readerParams, readerParams.getRowTransformer(), namespace, ranges, } else { final Iterator<GeoWaveRedisRow>[] iterators = new Iterator[readerParams.getAdapterIds().length]; int i = 0; for (final short adapterId : readerParams.getAdapterIds()) { final Pair<Boolean, Boolean> groupByRowAndSortByTime = RedisUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId); RedisUtils.getRowSetPrefix( namespace, readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName()); final Stream<Pair<ByteArray, Iterator<ScoredEntry<GeoWaveRedisPersistedRow>>>> streamIt = RedisUtils.getPartitions(client, setNamePrefix).stream().map(p -> {
public RocksDBReader( final RocksDBClient client, final ReaderParams<T> readerParams, final boolean async) { this.iterator = createIteratorForReader(client, readerParams, readerParams.getRowTransformer(), false); }
@Override public boolean apply(final DynamoDBRow input) { return Arrays.contains(readerParams.getAdapterIds(), input.getAdapterId()); } };
@Override public <T> Deleter<T> createDeleter(final ReaderParams<T> readerParams) { return new QueryAndDeleteByRow<>( createRowDeleter( readerParams.getIndex().getName(), readerParams.getAdapterStore(), readerParams.getInternalAdapterStore(), readerParams.getAdditionalAuthorizations()), // intentionally don't run this reader as async because it does // not work well while simultaneously deleting rows new RocksDBReader<>(client, readerParams, false)); }