public String getSegmentId() { if (segment != null) { return segment.getSegmentNo(); } else { return null; } }
/** * readCommittedScope provide Read Snapshot isolation. * @param segmentId * @param readCommittedScope * @return */ public static Segment toSegment(String segmentId, ReadCommittedScope readCommittedScope) { // SegmentId can be combination of segmentNo and segmentFileName. String[] split = segmentId.split("#"); if (split.length > 1) { return new Segment(split[0], split[1], readCommittedScope); } else if (split.length > 0) { return new Segment(split[0], null, readCommittedScope); } return new Segment(segmentId, null, readCommittedScope); }
/** * Converts to segment object * @param segmentId * @return */ public static Segment toSegment(String segmentId) { // SegmentId can be combination of segmentNo and segmentFileName. return toSegment(segmentId, null); }
@Override public SegmentRefreshInfo getCommittedSegmentRefreshInfo(Segment segment, UpdateVO updateVo) throws IOException { Map<String, SegmentRefreshInfo> snapShot = readCommittedIndexFileSnapShot.getSegmentTimestampUpdaterMap(); String segName; if (segment.getSegmentNo() != null) { segName = segment.getSegmentNo(); } else { segName = segment.getSegmentFileName(); } SegmentRefreshInfo segmentRefreshInfo = snapShot.get(segName); return segmentRefreshInfo; }
public static Set<TableBlockIndexUniqueIdentifier> getTableBlockUniqueIdentifiers(Segment segment) throws IOException { Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = new HashSet<>(); Map<String, String> indexFiles = segment.getCommittedIndexFile(); for (Map.Entry<String, String> indexFileEntry : indexFiles.entrySet()) { Path indexFile = new Path(indexFileEntry.getKey()); tableBlockIndexUniqueIdentifiers.add( new TableBlockIndexUniqueIdentifier(indexFile.getParent().toString(), indexFile.getName(), indexFileEntry.getValue(), segment.getSegmentNo())); } return tableBlockIndexUniqueIdentifiers; }
public boolean isRefreshNeeded(Segment seg, UpdateVO updateVo) throws IOException { SegmentRefreshInfo segmentRefreshInfo = seg.getSegmentRefreshInfo(updateVo); String segmentId = seg.getSegmentNo(); if (segmentRefreshTime.get(segmentId) == null && segmentRefreshInfo.getSegmentUpdatedTimestamp() != null) { segmentRefreshTime.put(segmentId, segmentRefreshInfo); return true; } if (manualSegmentRefresh.get(segmentId) != null && manualSegmentRefresh.get(segmentId)) { manualSegmentRefresh.put(segmentId, false); return true; } boolean isRefresh = segmentRefreshInfo.compare(segmentRefreshTime.get(segmentId)); if (isRefresh) { segmentRefreshTime.remove(segmentId); } return isRefresh; }
if (segmentsToBeDeleted.contains(new Segment(loadMetadata.getLoadName()))) { loadMetadata.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); loadMetadata.setModificationOrdeletionTimesStamp(Long.parseLong(updatedTimeStamp)); if (loadMetadata.getLoadName().equalsIgnoreCase(segName.getSegmentNo())) { .contains(Segment.toSegment(loadMetadata.getLoadName(), null))) { loadMetadata.setSegmentFile(loadMetadata.getLoadName() + "_" + updatedTimeStamp + CarbonTablePath.SEGMENT_EXT);
Segment segment = Segment.getSegment(targetSegment, carbonTable.getTablePath()); segmentList.add( new Segment(segment.getSegmentNo(), segment.getSegmentFileName(), readCommittedScope)); setSegmentsToAccess(job.getConfiguration(), segmentList);
@Override public List<CoarseGrainDataMap> getDataMaps(Segment segment) throws IOException { List<CoarseGrainDataMap> dataMaps = new ArrayList<>(); try { Set<String> shardPaths = segmentMap.get(segment.getSegmentNo()); if (shardPaths == null) { shardPaths = getAllShardPaths(getCarbonTable().getTablePath(), segment.getSegmentNo()); segmentMap.put(segment.getSegmentNo(), shardPaths); } Set<String> filteredShards = segment.getFilteredIndexShardNames(); for (String shard : shardPaths) { if (shard.endsWith(BloomIndexFileStore.MERGE_BLOOM_INDEX_SHARD_NAME) || filteredShards.contains(new File(shard).getName())) { // Filter out the tasks which are filtered through Main datamap. // for merge shard, shard pruning delay to be done before pruning blocklet BloomCoarseGrainDataMap bloomDM = new BloomCoarseGrainDataMap(); bloomDM.init(new BloomDataMapModel(shard, cache, segment.getConfiguration())); bloomDM.initIndexColumnConverters(getCarbonTable(), dataMapMeta.getIndexedColumns()); bloomDM.setFilteredShard(filteredShards); dataMaps.add(bloomDM); } } } catch (Exception e) { throw new IOException("Error occurs while init Bloom DataMap", e); } return dataMaps; }
public static boolean validateRangeOfSegmentList(String segmentId) throws InvalidConfigurationException { String[] values = segmentId.split(","); try { if (values.length == 0) { throw new InvalidConfigurationException( "carbon.input.segments.<database_name>.<table_name> value can't be empty."); } for (String value : values) { if (!value.equalsIgnoreCase("*")) { Segment segment = Segment.toSegment(value, null); Float aFloatValue = Float.parseFloat(segment.getSegmentNo()); if (aFloatValue < 0 || aFloatValue > Float.MAX_VALUE) { throw new InvalidConfigurationException( "carbon.input.segments.<database_name>.<table_name> value range should be greater " + "than 0 and less than " + Float.MAX_VALUE); } } } } catch (NumberFormatException nfe) { throw new InvalidConfigurationException( "carbon.input.segments.<database_name>.<table_name> value range is not valid"); } return true; } /**
CarbonTablePath.getMetadataPath(table.getTablePath())); for (CarbonInputSplit split : mbSplit.getAllSplits()) { String segmentId = Segment.getSegment(split.getSegmentId(), loadMetadataDetails).toString(); if (uniqueSegments.get(segmentId) == null) { segments.add(Segment.toSegment(segmentId, new TableStatusReadCommittedScope(table.getAbsoluteTableIdentifier(), loadMetadataDetails, FileFactory.getConfiguration())));
Map<String, Segment> segmentMap = new HashMap<>(); for (Segment segment : segments) { segmentMap.put(segment.getSegmentNo(), segment); Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment); tableBlockIndexUniqueIdentifierWrappers.add( new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable(), segment.getConfiguration()));
@Override public List<DataMapDistributable> toDistributable(Segment segment) { List<DataMapDistributable> dataMapDistributableList = new ArrayList<>(); Set<String> shardPaths = segmentMap.get(segment.getSegmentNo()); if (shardPaths == null) { shardPaths = getAllShardPaths(getCarbonTable().getTablePath(), segment.getSegmentNo()); segmentMap.put(segment.getSegmentNo(), shardPaths); } Set<String> filteredShards = segment.getFilteredIndexShardNames(); for (String shardPath : shardPaths) { // Filter out the tasks which are filtered through Main datamap. // for merge shard, shard pruning delay to be done before pruning blocklet if (shardPath.endsWith(BloomIndexFileStore.MERGE_BLOOM_INDEX_SHARD_NAME) || filteredShards.contains(new File(shardPath).getName())) { DataMapDistributable bloomDataMapDistributable = new BloomDataMapDistributable(shardPath, filteredShards); dataMapDistributableList.add(bloomDataMapDistributable); } } return dataMapDistributableList; }
/** * Prune the segments from the already pruned blocklets. * @param segments * @param prunedBlocklets */ private void pruneSegments(List<Segment> segments, List<ExtendedBlocklet> prunedBlocklets) { List<Segment> toBeRemovedSegments = new ArrayList<>(); for (Segment segment : segments) { boolean found = false; // Clear the old pruned index files if any present segment.getFilteredIndexShardNames().clear(); // Check the segment exist in any of the pruned blocklets. for (ExtendedBlocklet blocklet : prunedBlocklets) { if (blocklet.getSegmentId().equals(segment.toString())) { found = true; // Set the pruned index file to the segment for further pruning. String shardName = CarbonTablePath.getShardName(blocklet.getFilePath()); segment.setFilteredIndexShardName(shardName); } } // Add to remove segments list if not present in pruned blocklets. if (!found) { toBeRemovedSegments.add(segment); } } // Remove all segments which are already pruned from pruned blocklets segments.removeAll(toBeRemovedSegments); }
@Override public String toString() { final StringBuilder sb = new StringBuilder("TableBlockInfo{"); sb.append("filePath='").append(filePath).append('\''); sb.append(", blockOffset=").append(blockOffset); sb.append(", blockLength=").append(blockLength); sb.append(", segment='").append(segment.toString()).append('\''); sb.append(", blockletId='").append(blockletId).append('\''); sb.append(", locations=").append(Arrays.toString(locations)); sb.append('}'); return sb.toString(); } }
if (segmentsToBeDeleted.contains(new Segment(detail.getLoadName()))) { detail.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); } else if (segmentFilesTobeUpdated .contains(Segment.toSegment(detail.getLoadName(), null))) { detail.setSegmentFile( detail.getLoadName() + "_" + newMetaEntry.getUpdateStatusFileName()
.indexOf(Segment.toSegment(loadDetail.getLoadName(), null)); if (segmentFileIndex > -1) { loadDetail.setSegmentFile( segmentFilesToBeUpdated.get(segmentFileIndex).getSegmentFileName());
String tablePath, List<String> indexFileNamesTobeAdded, boolean readFileFooterFromCarbonDataFile, String uuid) throws IOException { Segment segment = Segment.getSegment(segmentId, tablePath); String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId); CarbonFile[] indexFiles; SegmentFileStore sfs = null; if (segment != null && segment.getSegmentFileName() != null) { sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName()); List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles(); indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]);
new SegmentFileStore(table.getTablePath(), segment.getSegmentFileName()) .dropPartitions(segment, partitionSpecs, uniqueId, tobeDeletedSegs, tobeUpdatedSegs); Segment.toSegmentList(tobeDeletedSegs, null), Segment.toSegmentList(tobeUpdatedSegs, null)); return uniqueId;
/** * return valid segment to access */ public Segment[] getSegmentsToAccess(JobContext job, ReadCommittedScope readCommittedScope) { String segmentString = job.getConfiguration().get(INPUT_SEGMENT_NUMBERS, ""); if (segmentString.trim().isEmpty()) { return new Segment[0]; } List<Segment> segments = Segment.toSegmentList(segmentString.split(","), readCommittedScope); return segments.toArray(new Segment[segments.size()]); }