@Override public List<DataMapDistributable> toDistributable(Segment segment) { List<DataMapDistributable> dataMapDistributableList = new ArrayList<>(); Set<String> shardPaths = segmentMap.get(segment.getSegmentNo()); if (shardPaths == null) { shardPaths = getAllShardPaths(getCarbonTable().getTablePath(), segment.getSegmentNo()); segmentMap.put(segment.getSegmentNo(), shardPaths); } Set<String> filteredShards = segment.getFilteredIndexShardNames(); for (String shardPath : shardPaths) { // Filter out the tasks which are filtered through Main datamap. // for merge shard, shard pruning delay to be done before pruning blocklet if (shardPath.endsWith(BloomIndexFileStore.MERGE_BLOOM_INDEX_SHARD_NAME) || filteredShards.contains(new File(shardPath).getName())) { DataMapDistributable bloomDataMapDistributable = new BloomDataMapDistributable(shardPath, filteredShards); dataMapDistributableList.add(bloomDataMapDistributable); } } return dataMapDistributableList; }
/** * Prune the segments from the already pruned blocklets. * @param segments * @param prunedBlocklets */ private void pruneSegments(List<Segment> segments, List<ExtendedBlocklet> prunedBlocklets) { List<Segment> toBeRemovedSegments = new ArrayList<>(); for (Segment segment : segments) { boolean found = false; // Clear the old pruned index files if any present segment.getFilteredIndexShardNames().clear(); // Check the segment exist in any of the pruned blocklets. for (ExtendedBlocklet blocklet : prunedBlocklets) { if (blocklet.getSegmentId().equals(segment.toString())) { found = true; // Set the pruned index file to the segment for further pruning. String shardName = CarbonTablePath.getShardName(blocklet.getFilePath()); segment.setFilteredIndexShardName(shardName); } } // Add to remove segments list if not present in pruned blocklets. if (!found) { toBeRemovedSegments.add(segment); } } // Remove all segments which are already pruned from pruned blocklets segments.removeAll(toBeRemovedSegments); }
@Override public List<CoarseGrainDataMap> getDataMaps(Segment segment) throws IOException { List<CoarseGrainDataMap> dataMaps = new ArrayList<>(); try { Set<String> shardPaths = segmentMap.get(segment.getSegmentNo()); if (shardPaths == null) { shardPaths = getAllShardPaths(getCarbonTable().getTablePath(), segment.getSegmentNo()); segmentMap.put(segment.getSegmentNo(), shardPaths); } Set<String> filteredShards = segment.getFilteredIndexShardNames(); for (String shard : shardPaths) { if (shard.endsWith(BloomIndexFileStore.MERGE_BLOOM_INDEX_SHARD_NAME) || filteredShards.contains(new File(shard).getName())) { // Filter out the tasks which are filtered through Main datamap. // for merge shard, shard pruning delay to be done before pruning blocklet BloomCoarseGrainDataMap bloomDM = new BloomCoarseGrainDataMap(); bloomDM.init(new BloomDataMapModel(shard, cache, segment.getConfiguration())); bloomDM.initIndexColumnConverters(getCarbonTable(), dataMapMeta.getIndexedColumns()); bloomDM.setFilteredShard(filteredShards); dataMaps.add(bloomDM); } } } catch (Exception e) { throw new IOException("Error occurs while init Bloom DataMap", e); } return dataMaps; }
/** * Get all distributable objects of a segmentid */ @Override public List<DataMapDistributable> toDistributable(Segment segment) { List<DataMapDistributable> lstDataMapDistribute = new ArrayList<>(); CarbonFile[] indexDirs = getAllIndexDirs(tableIdentifier.getTablePath(), segment.getSegmentNo()); if (segment.getFilteredIndexShardNames().size() == 0) { for (CarbonFile indexDir : indexDirs) { DataMapDistributable luceneDataMapDistributable = new LuceneDataMapDistributable(tableIdentifier.getTablePath(), indexDir.getAbsolutePath()); lstDataMapDistribute.add(luceneDataMapDistributable); } return lstDataMapDistribute; } for (CarbonFile indexDir : indexDirs) { // Filter out the tasks which are filtered through CG datamap. if (getDataMapLevel() != DataMapLevel.FG && !segment.getFilteredIndexShardNames().contains(indexDir.getName())) { continue; } DataMapDistributable luceneDataMapDistributable = new LuceneDataMapDistributable( CarbonTablePath.getSegmentPath(tableIdentifier.getTablePath(), segment.getSegmentNo()), indexDir.getAbsolutePath()); lstDataMapDistribute.add(luceneDataMapDistributable); } return lstDataMapDistribute; }