public static ByteArrayRange fromRowRange(final GeoWaveRowRange range) { if ((range.getPartitionKey() == null) || (range.getPartitionKey().length == 0)) { final byte[] startKey = (range.getStartSortKey() == null) ? null : range.getStartSortKey(); final byte[] endKey = (range.getEndSortKey() == null) ? null : range.getEndSortKey(); return new ByteArrayRange(startKey, endKey); } else { final byte[] startKey = (range.getStartSortKey() == null) ? range.getPartitionKey() : ArrayUtils.addAll(range.getPartitionKey(), range.getStartSortKey()); final byte[] endKey = (range.getEndSortKey() == null) ? ByteArrayUtils.getNextPrefix(range.getPartitionKey()) : ArrayUtils.addAll(range.getPartitionKey(), range.getEndSortKey()); return new ByteArrayRange(startKey, endKey); } }
protected static BigInteger getStart(final GeoWaveRowRange range, final int cardinality) { final byte[] start = range.getStartSortKey(); byte[] startBytes; if (!range.isInfiniteStartSortKey() && (start != null)) { startBytes = extractBytes(start, cardinality); } else { startBytes = extractBytes(new byte[] {}, cardinality); } return new BigInteger(startBytes); }
protected static BigInteger getEnd(final GeoWaveRowRange range, final int cardinality) { final byte[] end = range.getEndSortKey(); byte[] endBytes; if (!range.isInfiniteStopSortKey() && (end != null)) { endBytes = extractBytes(end, cardinality); } else { endBytes = extractBytes(new byte[] {}, cardinality, true); } return new BigInteger(endBytes); }
public void readFields(final DataInput in) throws IOException, InstantiationException, IllegalAccessException { final boolean nullRange = in.readBoolean(); if (nullRange) { range = null; } else { range = new GeoWaveRowRange(); range.readFields(in); } location = in.readUTF(); cardinality = in.readDouble(); }
protected static double getRangeLength(final GeoWaveRowRange range) { if ((range == null) || (range.getStartSortKey() == null) || (range.getEndSortKey() == null)) { return 1; } final byte[] start = range.getStartSortKey(); final byte[] end = range.getEndSortKey(); final int maxDepth = Math.max(end.length, start.length); final BigInteger startBI = new BigInteger(extractBytes(start, maxDepth)); final BigInteger endBI = new BigInteger(extractBytes(end, maxDepth)); return endBI.subtract(startBI).doubleValue(); }
private CloseableIterator<T> createIteratorForRecordReader( final RocksDBClient client, final RecordReaderParams recordReaderParams) { final GeoWaveRowRange range = recordReaderParams.getRowRange(); final byte[] startKey = range.isInfiniteStartSortKey() ? null : range.getStartSortKey(); final byte[] stopKey = range.isInfiniteStopSortKey() ? null : range.getEndSortKey(); final SinglePartitionQueryRanges partitionRange = new SinglePartitionQueryRanges( range.getPartitionKey(), Collections.singleton(new ByteArrayRange(startKey, stopKey))); final Set<String> authorizations = Sets.newHashSet(recordReaderParams.getAdditionalAuthorizations()); return createIterator( client, (RangeReaderParams<T>) recordReaderParams, (GeoWaveRowIteratorTransformer<T>) GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER, Collections.singleton(partitionRange), authorizations, // there should already be sufficient parallelism created by // input splits for record reader use cases false); }
final byte[] start = rangeLocationPair.getRange().getStartSortKey(); final byte[] end = rangeLocationPair.getRange().getEndSortKey(); final double cdfStart = start == null ? 0.0 : stats.cdf(start); if (Arrays.equals(end, splitKey) && !rangeLocationPair.getRange().isEndSortKeyInclusive()) { rangeLocationPair = new RangeLocationPair( new GeoWaveRowRange( rangeLocationPair.getRange().getPartitionKey(), rangeLocationPair.getRange().getStartSortKey(), splitKey, rangeLocationPair.getRange().isStartSortKeyInclusive(), endKeyInclusive), location, stats.cardinality(rangeLocationPair.getRange().getStartSortKey(), splitKey)); return null; final RangeLocationPair newPair = new RangeLocationPair( new GeoWaveRowRange( rangeLocationPair.getRange().getPartitionKey(), rangeLocationPair.getRange().getStartSortKey(), splitKey, rangeLocationPair.getRange().isStartSortKeyInclusive(), endKeyInclusive), location, stats.cardinality(rangeLocationPair.getRange().getStartSortKey(), splitKey)); new GeoWaveRowRange(
protected static GeoWaveRowRange rangeIntersection( final GeoWaveRowRange thisRange, final GeoWaveRowRange otherRange) { final ByteArrayRange thisByteArrayRange = new ByteArrayRange(thisRange.getStartSortKey(), thisRange.getEndSortKey()); final ByteArrayRange otherByteArrayRange = new ByteArrayRange(otherRange.getStartSortKey(), otherRange.getEndSortKey()); final ByteArrayRange overlappingRange = thisByteArrayRange.intersection(otherByteArrayRange); return new GeoWaveRowRange( null, overlappingRange.getStart(), overlappingRange.getEnd(), true, false); } }
for (final ByteArray partitionKey : partitionKeys) { final GeoWaveRowRange gwRange = new GeoWaveRowRange(partitionKey.getBytes(), null, null, true, true); final double cardinality = getCardinality( new RangeLocationPair(new GeoWaveRowRange(null, null, null, true, false), 0.0)); statsStore, statsCache, new ByteArray(gwRange.getPartitionKey()), authorizations), gwRange);
Pair.of( next.index, new ByteArray(next.rangeLocationPair.getRange().getPartitionKey()))), currentCardinality, targetCardinality);
return new GeoWaveRowRange(null, startRow, stopRow, true, false); } else { byte[] partitionKey; boolean partitionKeyDiffers = false; if ((startRow == null) && (stopRow == null)) { return new GeoWaveRowRange(null, null, null, true, true); } else if (startRow != null) { partitionKey = ArrayUtils.subarray(startRow, 0, partitionKeyLength); partitionKey = ArrayUtils.subarray(stopRow, 0, partitionKeyLength); return new GeoWaveRowRange( partitionKey, startRow == null ? null
private CloseableIterator<T> createIteratorForRecordReader( final RedissonClient client, final Compression compression, final RecordReaderParams recordReaderParams, final String namespace, final boolean visibilityEnabled) { final GeoWaveRowRange range = recordReaderParams.getRowRange(); final byte[] startKey = range.isInfiniteStartSortKey() ? null : range.getStartSortKey(); final byte[] stopKey = range.isInfiniteStopSortKey() ? null : range.getEndSortKey(); final SinglePartitionQueryRanges partitionRange = new SinglePartitionQueryRanges( range.getPartitionKey(), Collections.singleton(new ByteArrayRange(startKey, stopKey))); final Set<String> authorizations = Sets.newHashSet(recordReaderParams.getAdditionalAuthorizations()); return createIterator( client, compression, (RangeReaderParams<T>) recordReaderParams, (GeoWaveRowIteratorTransformer<T>) GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER, namespace, Collections.singleton(partitionRange), authorizations, visibilityEnabled, // there should already be sufficient parallelism created by // input splits for record reader use cases false); }
protected double getCardinality( final RowRangeHistogramStatistics<?> rangeStats, final GeoWaveRowRange range) { if (range == null) { if (rangeStats != null) { return rangeStats.getTotalCount(); } else { // with an infinite range and no histogram we have no info to // base a cardinality on return 0; } } return rangeStats == null ? 0.0 : rangeStats.cardinality(range.getStartSortKey(), range.getEndSortKey()); }
statsStore, statsCache, new ByteArray(gwRange.getPartitionKey()), authorizations), gwRange);
protected void initRecordScanner() { final String tableName = operations.getQualifiedTableName(recordReaderParams.getIndex().getName()); final ArrayList<Short> adapterIds = Lists.newArrayList(); if ((recordReaderParams.getAdapterIds() != null) && (recordReaderParams.getAdapterIds().length > 0)) { for (final Short adapterId : recordReaderParams.getAdapterIds()) { adapterIds.add(adapterId); } } final List<QueryRequest> requests = new ArrayList<>(); final GeoWaveRowRange range = recordReaderParams.getRowRange(); for (final Short adapterId : adapterIds) { final byte[] startKey = range.isInfiniteStartSortKey() ? null : range.getStartSortKey(); final byte[] stopKey = range.isInfiniteStopSortKey() ? null : range.getEndSortKey(); requests.add( getQuery( tableName, range.getPartitionKey(), new ByteArrayRange(startKey, stopKey), adapterId)); } startRead(requests, tableName, recordReaderParams.isClientsideRowMerging(), false); }
protected static byte[] getMidpoint(final GeoWaveRowRange range) { if ((range.getStartSortKey() == null) || (range.getEndSortKey() == null)) { return null; } final byte[] start = range.getStartSortKey(); final byte[] end = range.getEndSortKey(); if (Arrays.equals(start, end)) { return null; } final int maxDepth = Math.max(end.length, start.length); final BigInteger startBI = new BigInteger(extractBytes(start, maxDepth)); final BigInteger endBI = new BigInteger(extractBytes(end, maxDepth)); final BigInteger rangeBI = endBI.subtract(startBI); if (rangeBI.equals(BigInteger.ZERO) || rangeBI.equals(BigInteger.ONE)) { return end; } final byte[] valueBytes = rangeBI.divide(TWO).add(startBI).toByteArray(); final byte[] bytes = new byte[valueBytes.length - 2]; System.arraycopy(valueBytes, 2, bytes, 0, bytes.length); return bytes; }
protected void initRecordScanner() { final short[] adapterIds = recordReaderParams.getAdapterIds() != null ? recordReaderParams.getAdapterIds() : new short[0]; final GeoWaveRowRange range = recordReaderParams.getRowRange(); final byte[] startKey = range.isInfiniteStartSortKey() ? null : range.getStartSortKey(); final byte[] stopKey = range.isInfiniteStopSortKey() ? null : range.getEndSortKey(); final SinglePartitionQueryRanges partitionRange = new SinglePartitionQueryRanges( range.getPartitionKey(), Collections.singleton(new ByteArrayRange(startKey, stopKey))); final Set<String> authorizations = Sets.newHashSet(recordReaderParams.getAdditionalAuthorizations()); iterator = operations.getBatchedRangeRead( recordReaderParams.getIndex().getName(), adapterIds, Collections.singleton(partitionRange), DataStoreUtils.isMergingIteratorRequired(recordReaderParams, visibilityEnabled), rowTransformer, new ClientVisibilityFilter(authorizations)).results(); }
private static float getProgressForRange( final GeoWaveRowRange range, final GeoWaveInputKey currentKey) { if (currentKey == null) { return 0f; } if ((range != null) && (range.getStartSortKey() != null) && (range.getEndSortKey() != null) && (currentKey.getGeoWaveKey() != null)) { // TODO GEOWAVE-1018 this doesn't account for partition keys at all // just look at the row progress return getProgressForRange( range.getStartSortKey(), range.getEndSortKey(), GeoWaveKey.getCompositeId(currentKey.getGeoWaveKey())); } // if we can't figure it out, then claim no progress return 0f; }