@Override public String toString() { final StringBuilder sb = new StringBuilder("TableBlockInfo{"); sb.append("filePath='").append(filePath).append('\''); sb.append(", blockOffset=").append(blockOffset); sb.append(", blockLength=").append(blockLength); sb.append(", segment='").append(segment.toString()).append('\''); sb.append(", blockletId='").append(blockletId).append('\''); sb.append(", locations=").append(Arrays.toString(locations)); sb.append('}'); return sb.toString(); } }
@Override public void write(DataOutput out) throws IOException { super.write(out); out.writeUTF(segment.toString()); out.writeShort(version.number()); out.writeUTF(bucketId); out.writeUTF(blockletId); out.writeInt(invalidSegments.size()); for (String invalidSegment : invalidSegments) { out.writeUTF(invalidSegment); } out.writeInt(null != deleteDeltaFiles ? deleteDeltaFiles.length : 0); if (null != deleteDeltaFiles) { for (int i = 0; i < deleteDeltaFiles.length; i++) { out.writeUTF(deleteDeltaFiles[i]); } } out.writeBoolean(detailInfo != null); if (detailInfo != null) { detailInfo.write(out); } out.writeBoolean(dataMapWritePath != null); if (dataMapWritePath != null) { out.writeUTF(dataMapWritePath); } }
private List<ExtendedBlocklet> pruneWithFilter(List<Segment> segments, FilterResolverIntf filterExp, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets, Map<Segment, List<DataMap>> dataMaps) throws IOException { for (Segment segment : segments) { List<Blocklet> pruneBlocklets = new ArrayList<>(); SegmentProperties segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment); for (DataMap dataMap : dataMaps.get(segment)) { pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions)); } blocklets.addAll( addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.toString())); } return blocklets; }
private List<ExtendedBlocklet> pruneWithoutFilter(List<Segment> segments, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets) throws IOException { for (Segment segment : segments) { List<Blocklet> allBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions); blocklets.addAll( addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(allBlocklets, segment), segment.toString())); } return blocklets; }
/** * Prune the segments from the already pruned blocklets. * @param segments * @param prunedBlocklets */ private void pruneSegments(List<Segment> segments, List<ExtendedBlocklet> prunedBlocklets) { List<Segment> toBeRemovedSegments = new ArrayList<>(); for (Segment segment : segments) { boolean found = false; // Clear the old pruned index files if any present segment.getFilteredIndexShardNames().clear(); // Check the segment exist in any of the pruned blocklets. for (ExtendedBlocklet blocklet : prunedBlocklets) { if (blocklet.getSegmentId().equals(segment.toString())) { found = true; // Set the pruned index file to the segment for further pruning. String shardName = CarbonTablePath.getShardName(blocklet.getFilePath()); segment.setFilteredIndexShardName(shardName); } } // Add to remove segments list if not present in pruned blocklets. if (!found) { toBeRemovedSegments.add(segment); } } // Remove all segments which are already pruned from pruned blocklets segments.removeAll(toBeRemovedSegments); }
blocklets.addAll(addSegmentId( blockletDetailsFetcher.getExtendedBlocklets(entry.getValue(), entry.getKey()), entry.getKey().toString())); } catch (IOException e) { throw new RuntimeException(e);
/** * Pass the valid segments and prune the datamap using filter expression * * @param segments * @param filterExp * @return */ public List<ExtendedBlocklet> prune(List<Segment> segments, Expression filterExp, List<PartitionSpec> partitions) throws IOException { List<ExtendedBlocklet> blocklets = new ArrayList<>(); SegmentProperties segmentProperties; Map<Segment, List<DataMap>> dataMaps = dataMapFactory.getDataMaps(segments); for (Segment segment : segments) { List<Blocklet> pruneBlocklets = new ArrayList<>(); // if filter is not passed then return all the blocklets if (filterExp == null) { pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions); } else { segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment); for (DataMap dataMap : dataMaps.get(segment)) { pruneBlocklets .addAll(dataMap.prune(filterExp, segmentProperties, partitions, identifier)); } } blocklets.addAll(addSegmentId( blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.toString())); } return blocklets; }
CarbonTablePath.getMetadataPath(table.getTablePath())); for (CarbonInputSplit split : mbSplit.getAllSplits()) { String segmentId = Segment.getSegment(split.getSegmentId(), loadMetadataDetails).toString(); if (uniqueSegments.get(segmentId) == null) { segments.add(Segment.toSegment(segmentId,
public static List<TableBlockInfo> createBlocks(List<CarbonInputSplit> splitList) { List<TableBlockInfo> tableBlockInfoList = new ArrayList<>(); for (CarbonInputSplit split : splitList) { BlockletInfos blockletInfos = new BlockletInfos(split.getNumberOfBlocklets(), 0, split.getNumberOfBlocklets()); try { TableBlockInfo blockInfo = new TableBlockInfo(split.getPath().toString(), split.blockletId, split.getStart(), split.getSegment().toString(), split.getLocations(), split.getLength(), blockletInfos, split.getVersion(), split.getDeleteDeltaFiles()); blockInfo.setDetailInfo(split.getDetailInfo()); blockInfo.setDataMapWriterPath(split.dataMapWritePath); blockInfo.setBlockOffset(split.getDetailInfo().getBlockFooterOffset()); tableBlockInfoList.add(blockInfo); } catch (IOException e) { throw new RuntimeException("fail to get location of split: " + split, e); } } return tableBlockInfoList; }
serializer.serializeBlocklet((FineGrainBlocklet) blocklet, blockletwritePath); detailedBlocklet.setSegmentId(distributable.getSegment().toString()); detailedBlocklets.add(detailedBlocklet);
public static TableBlockInfo getTableBlockInfo(CarbonInputSplit inputSplit) { BlockletInfos blockletInfos = new BlockletInfos(inputSplit.getNumberOfBlocklets(), 0, inputSplit.getNumberOfBlocklets()); try { TableBlockInfo blockInfo = new TableBlockInfo(inputSplit.getPath().toString(), inputSplit.blockletId, inputSplit.getStart(), inputSplit.getSegment().toString(), inputSplit.getLocations(), inputSplit.getLength(), blockletInfos, inputSplit.getVersion(), inputSplit.getDeleteDeltaFiles()); blockInfo.setDetailInfo(inputSplit.getDetailInfo()); blockInfo.setBlockOffset(inputSplit.getDetailInfo().getBlockFooterOffset()); return blockInfo; } catch (IOException e) { throw new RuntimeException("fail to get location of split: " + inputSplit, e); } }