private List<ExtendedBlocklet> pruneWithoutFilter(List<Segment> segments, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets) throws IOException { for (Segment segment : segments) { List<Blocklet> allBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions); blocklets.addAll( addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(allBlocklets, segment), segment.toString())); } return blocklets; }
private List<ExtendedBlocklet> pruneWithFilter(List<Segment> segments, FilterResolverIntf filterExp, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets, Map<Segment, List<DataMap>> dataMaps) throws IOException { for (Segment segment : segments) { List<Blocklet> pruneBlocklets = new ArrayList<>(); SegmentProperties segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment); for (DataMap dataMap : dataMaps.get(segment)) { pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions)); } blocklets.addAll( addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.toString())); } return blocklets; }
blocklets.addAll(addSegmentId( blockletDetailsFetcher.getExtendedBlocklets(entry.getValue(), entry.getKey()), entry.getKey().toString()));
/** * Pass the valid segments and prune the datamap using filter expression * * @param segments * @param filterExp * @return */ public List<ExtendedBlocklet> prune(List<Segment> segments, Expression filterExp, List<PartitionSpec> partitions) throws IOException { List<ExtendedBlocklet> blocklets = new ArrayList<>(); SegmentProperties segmentProperties; Map<Segment, List<DataMap>> dataMaps = dataMapFactory.getDataMaps(segments); for (Segment segment : segments) { List<Blocklet> pruneBlocklets = new ArrayList<>(); // if filter is not passed then return all the blocklets if (filterExp == null) { pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions); } else { segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment); for (DataMap dataMap : dataMaps.get(segment)) { pruneBlocklets .addAll(dataMap.prune(filterExp, segmentProperties, partitions, identifier)); } } blocklets.addAll(addSegmentId( blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.toString())); } return blocklets; }