/** * Filter the given iterable of objects by removing any object whose ShardSpec, obtained from the converter function, * does not fit in the RangeSet of the dimFilter {@link DimFilter#getDimensionRangeSet(String)}. The returned set * contains the filtered objects in the same order as they appear in input. * * If you plan to call this multiple times with the same dimFilter, consider using * {@link #filterShards(DimFilter, Iterable, Function, Map)} instead with a cached map * * @param dimFilter The filter to use * @param input The iterable of objects to be filtered * @param converter The function to convert T to ShardSpec that can be filtered by * @param <T> This can be any type, as long as transform function is provided to convert this to ShardSpec * @return The set of filtered object, in the same order as input */ public static <T> Set<T> filterShards(DimFilter dimFilter, Iterable<T> input, Function<T, ShardSpec> converter) { return filterShards(dimFilter, input, converter, new HashMap<String, Optional<RangeSet<String>>>()); }
@Override public byte[] getCacheKey() { return DimFilterUtils.computeCacheKey(DimFilterUtils.OR_CACHE_ID, fields); }
private void assertFilterResult(DimFilter filter, Iterable<ShardSpec> input, Set<ShardSpec> expected) { Set<ShardSpec> result = DimFilterUtils.filterShards(filter, input, CONVERTER); Assert.assertEquals(expected, result); Map<String, Optional<RangeSet<String>>> dimensionRangeMap = new HashMap<>(); result = new HashSet<>(); for (ShardSpec shard : input) { result.addAll(DimFilterUtils.filterShards(filter, ImmutableList.of(shard), CONVERTER, dimensionRangeMap)); } Assert.assertEquals(expected, result); }
@Override public byte[] getCacheKey() { return DimFilterUtils.computeCacheKey(DimFilterUtils.AND_CACHE_ID, fields); }
private Set<ServerToSegment> computeSegmentsToQuery(TimelineLookup<String, ServerSelector> timeline) { final List<TimelineObjectHolder<String, ServerSelector>> serversLookup = toolChest.filterSegments( query, intervals.stream().flatMap(i -> timeline.lookup(i).stream()).collect(Collectors.toList()) ); final Set<ServerToSegment> segments = new LinkedHashSet<>(); final Map<String, Optional<RangeSet<String>>> dimensionRangeCache = new HashMap<>(); // Filter unneeded chunks based on partition dimension for (TimelineObjectHolder<String, ServerSelector> holder : serversLookup) { final Set<PartitionChunk<ServerSelector>> filteredChunks = DimFilterUtils.filterShards( query.getFilter(), holder.getObject(), partitionChunk -> partitionChunk.getObject().getSegment().getShardSpec(), dimensionRangeCache ); for (PartitionChunk<ServerSelector> chunk : filteredChunks) { ServerSelector server = chunk.getObject(); final SegmentDescriptor segment = new SegmentDescriptor( holder.getInterval(), holder.getVersion(), chunk.getChunkNumber() ); segments.add(new ServerToSegment(server, segment)); } } return segments; }
@Override public byte[] getCacheKey() { return DimFilterUtils.computeCacheKey(DimFilterUtils.AND_CACHE_ID, fields); }
/** * Filter the given iterable of objects by removing any object whose ShardSpec, obtained from the converter function, * does not fit in the RangeSet of the dimFilter {@link DimFilter#getDimensionRangeSet(String)}. The returned set * contains the filtered objects in the same order as they appear in input. * * If you plan to call this multiple times with the same dimFilter, consider using * {@link #filterShards(DimFilter, Iterable, Function, Map)} instead with a cached map * * @param dimFilter The filter to use * @param input The iterable of objects to be filtered * @param converter The function to convert T to ShardSpec that can be filtered by * @param <T> This can be any type, as long as transform function is provided to convert this to ShardSpec * @return The set of filtered object, in the same order as input */ public static <T> Set<T> filterShards(DimFilter dimFilter, Iterable<T> input, Function<T, ShardSpec> converter) { return filterShards(dimFilter, input, converter, new HashMap<String, Optional<RangeSet<String>>>()); }
@Override public byte[] getCacheKey() { return DimFilterUtils.computeCacheKey(DimFilterUtils.OR_CACHE_ID, fields); }
private Set<ServerToSegment> computeSegmentsToQuery(TimelineLookup<String, ServerSelector> timeline) { final List<TimelineObjectHolder<String, ServerSelector>> serversLookup = toolChest.filterSegments( query, query.getIntervals().stream().flatMap(i -> timeline.lookup(i).stream()).collect(Collectors.toList()) ); final Set<ServerToSegment> segments = Sets.newLinkedHashSet(); final Map<String, Optional<RangeSet<String>>> dimensionRangeCache = Maps.newHashMap(); // Filter unneeded chunks based on partition dimension for (TimelineObjectHolder<String, ServerSelector> holder : serversLookup) { final Set<PartitionChunk<ServerSelector>> filteredChunks = DimFilterUtils.filterShards( query.getFilter(), holder.getObject(), partitionChunk -> partitionChunk.getObject().getSegment().getShardSpec(), dimensionRangeCache ); for (PartitionChunk<ServerSelector> chunk : filteredChunks) { ServerSelector server = chunk.getObject(); final SegmentDescriptor segment = new SegmentDescriptor( holder.getInterval(), holder.getVersion(), chunk.getChunkNumber() ); segments.add(new ServerToSegment(server, segment)); } } return segments; }