private boolean isScanRequired(BlockletHeader header) { if (filter != null && header.getBlocklet_index() != null) { BlockletMinMaxIndex minMaxIndex = CarbonMetadataUtil.convertExternalMinMaxIndex( header.getBlocklet_index().getMin_max_index()); if (minMaxIndex != null) { BitSet bitSet = filter .isScanRequired(minMaxIndex.getMaxValues(), minMaxIndex.getMinValues(), minMaxIndex.getIsMinMaxSet()); if (bitSet.isEmpty()) { return false; } else { return true; } } } return true; }
/** * merge new blocklet index and old file index to create new file index */ private static void updateStreamFileIndex(Map<String, StreamFileIndex> indexMap, String indexPath, FileFactory.FileType fileType, DataType[] msrDataTypes ) throws IOException { List<BlockIndex> blockIndexList = readIndexFile(indexPath, fileType); for (BlockIndex blockIndex : blockIndexList) { BlockletMinMaxIndex fileIndex = CarbonMetadataUtil .convertExternalMinMaxIndex(blockIndex.getBlock_index().getMin_max_index()); StreamFileIndex blockletIndex = indexMap.get(blockIndex.getFile_name()); if (blockletIndex == null) { // should index all stream file indexMap.put(blockIndex.getFile_name(), new StreamFileIndex(blockIndex.getFile_name(), fileIndex, blockIndex.getNum_rows())); } else { // merge minMaxIndex into StreamBlockIndex blockletIndex.setRowCount(blockletIndex.getRowCount() + blockIndex.getNum_rows()); mergeBatchMinMax(blockletIndex, fileIndex, msrDataTypes); } } }
&& blockIndex.getBlock_index().getMin_max_index() != null) { streamFile.setMinMaxIndex(CarbonMetadataUtil .convertExternalMinMaxIndex(blockIndex.getBlock_index().getMin_max_index()));