@Override public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) { double seg1Id = Double.parseDouble(seg1.getLoadName()); double seg2Id = Double.parseDouble(seg2.getLoadName()); return Double.compare(seg1Id, seg2Id); } });
private static LoadMetadataDetails getCurrentLoadStatusOfSegment(String segmentId, String metadataPath) { LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath); for (LoadMetadataDetails oneLoad : currentDetails) { if (oneLoad.getLoadName().equalsIgnoreCase(segmentId)) { return oneLoad; } } return null; }
/** * Form the Name of the New Merge Folder * * @param segmentsToBeMergedList * @return */ public static String getMergedLoadName(List<LoadMetadataDetails> segmentsToBeMergedList) { String firstSegmentName = segmentsToBeMergedList.get(0).getLoadName(); if (firstSegmentName.contains(".")) { String beforeDecimal = firstSegmentName.substring(0, firstSegmentName.indexOf(".")); String afterDecimal = firstSegmentName.substring(firstSegmentName.indexOf(".") + 1); int fraction = Integer.parseInt(afterDecimal) + 1; String mergedSegmentName = beforeDecimal + "." + fraction; return CarbonCommonConstants.LOAD_FOLDER + mergedSegmentName; } else { String mergeName = firstSegmentName + "." + 1; return CarbonCommonConstants.LOAD_FOLDER + mergeName; } }
/** * Get the segment object corresponding to segmentNo * @param segmentNo * @param loadMetadataDetails * @return */ public static Segment getSegment(String segmentNo, LoadMetadataDetails[] loadMetadataDetails) { for (LoadMetadataDetails details: loadMetadataDetails) { if (details.getLoadName().equals(segmentNo)) { return new Segment(details.getLoadName(), details.getSegmentFile()); } } return null; }
for (int i = 0; i < loadMetadataDetails.length; i++) { try { int loadCount = Integer.parseInt(loadMetadataDetails[i].getLoadName()); if (newSegmentId < loadCount) { newSegmentId = loadCount; String loadName = loadMetadataDetails[i].getLoadName(); if (loadName.contains(".")) { int loadCount = Integer.parseInt(loadName.split("\\.")[0]);
/** * Get the number of invisible segment info from segment info list. */ public static int countInvisibleSegments( LoadMetadataDetails[] segmentList, int maxSegmentId) { int invisibleSegmentCnt = 0; if (segmentList.length != 0) { for (LoadMetadataDetails eachSeg : segmentList) { // can not remove segment 0, there are some info will be used later // for example: updateStatusFileName // also can not remove the max segment id, // otherwise will impact the generation of segment id if (!eachSeg.getLoadName().equalsIgnoreCase("0") && !eachSeg.getLoadName().equalsIgnoreCase(String.valueOf(maxSegmentId)) && eachSeg.getVisibility().equalsIgnoreCase("false")) { invisibleSegmentCnt += 1; } } } return invisibleSegmentCnt; }
/** * This API will return the update status file name. * @param segmentList * @return */ public String getUpdateStatusFileName(LoadMetadataDetails[] segmentList) { if (segmentList.length == 0) { return ""; } else { for (LoadMetadataDetails eachSeg : segmentList) { // file name stored in 0th segment. if (eachSeg.getLoadName().equalsIgnoreCase("0")) { return eachSeg.getUpdateStatusFileName(); } } } return ""; }
/** * This method will return the list of loads which are specified by user in SQL. * * @param listOfSegments * @param segmentIds * @return */ private static List<LoadMetadataDetails> identitySegmentsToBeMergedBasedOnSpecifiedSegments( List<LoadMetadataDetails> listOfSegments, Set<String> segmentIds) throws MalformedCarbonCommandException { Map<String, LoadMetadataDetails> specifiedSegments = new LinkedHashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); for (LoadMetadataDetails detail : listOfSegments) { if (segmentIds.contains(detail.getLoadName())) { specifiedSegments.put(detail.getLoadName(), detail); } } // all requested segments should exist and be valid for (String segmentId : segmentIds) { if (!specifiedSegments.containsKey(segmentId) || !isSegmentValid(specifiedSegments.get(segmentId))) { String errMsg = String.format("Segment %s does not exist or is not valid", segmentId); LOGGER.error(errMsg); throw new MalformedCarbonCommandException(errMsg); } } return new ArrayList<>(specifiedSegments.values()); }
@Override public boolean accept(CarbonFile path) { String segmentId = CarbonTablePath.DataFileUtil.getSegmentIdFromPath(path.getAbsolutePath() + "/dummy"); boolean found = false; for (int j = 0; j < details.length; j++) { if (details[j].getLoadName().equals(segmentId)) { found = true; break; } } return !found; } });
/** * returns segment path * * @param identifier * @param oneLoad * @return */ private static String getSegmentPath(AbsoluteTableIdentifier identifier, LoadMetadataDetails oneLoad) { String segmentId = oneLoad.getLoadName(); return CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId); }
/** * For getting the comma separated valid segments for merging. * * @param loadMetadataDetails * @return */ public static List<Segment> getValidSegments(List<LoadMetadataDetails> loadMetadataDetails) { List<Segment> segments = new ArrayList<>(); for (LoadMetadataDetails segment : loadMetadataDetails) { //check if this load is an already merged load. if (null != segment.getMergedLoadName()) { segments.add(Segment.toSegment(segment.getMergedLoadName(), null)); } else { segments.add(Segment.toSegment(segment.getLoadName(), null)); } } return segments; }
if (i < oldSegmentsLength) { LoadMetadataDetails oldSegment = oldList[i]; if (newSegment.getLoadName().equalsIgnoreCase("0") || newSegment.getLoadName().equalsIgnoreCase(String.valueOf(maxSegmentId))) { newSegment.setVisibility(oldSegment.getVisibility()); arrayOfVisibleSegments[visibleIdx] = newSegment;
private static void addToStaleFolders(AbsoluteTableIdentifier identifier, List<CarbonFile> staleFolders, LoadMetadataDetails entry) throws IOException { String path = CarbonTablePath.getSegmentPath( identifier.getTablePath(), entry.getLoadName()); // add to the deletion list only if file exist else HDFS file system will throw // exception while deleting the file if file path does not exist if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) { staleFolders.add(FileFactory.getCarbonFile(path)); } }
CarbonFile[] allSegmentFiles, SegmentUpdateStatusManager updateStatusManager) { CarbonFile[] invalidUpdateDeltaFiles = updateStatusManager .getUpdateDeltaFilesList(segment.getLoadName(), false, CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles, true); .getUpdateDeltaFilesList(segment.getLoadName(), false, CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles, true);
/** * method to identify the segments qualified for merging in case of IUD Compaction. * * @param segments * @param carbonLoadModel * @return */ private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnIUD( List<LoadMetadataDetails> segments, CarbonLoadModel carbonLoadModel) { List<LoadMetadataDetails> validSegments = new ArrayList<>(segments.size()); AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(); int numberUpdateDeltaFilesThreshold = CarbonProperties.getInstance().getNoUpdateDeltaFilesThresholdForIUDCompaction(); for (LoadMetadataDetails seg : segments) { if ((isSegmentValid(seg)) && checkUpdateDeltaFilesInSeg( new Segment(seg.getLoadName(), seg.getSegmentFile()), absoluteTableIdentifier, carbonLoadModel.getSegmentUpdateStatusManager(), numberUpdateDeltaFilesThreshold)) { validSegments.add(seg); } } return validSegments; }
/** * Return true if insert overwrite is in progress for specified table */ public static Boolean isOverwriteInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
/** * Return true if any load or insert overwrite is in progress for specified table */ public static Boolean isLoadInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_IN_PROGRESS || segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
/** * Returns the invalid timestamp range of a segment. * @param segmentId * @return */ public UpdateVO getInvalidTimestampRange(String segmentId) { UpdateVO range = new UpdateVO(); for (LoadMetadataDetails segment : segmentDetails) { if (segment.getLoadName().equalsIgnoreCase(segmentId)) { range.setSegmentId(segmentId); range.setFactTimestamp(segment.getLoadStartTime()); if (!segment.getUpdateDeltaStartTimestamp().isEmpty() && !segment .getUpdateDeltaEndTimestamp().isEmpty()) { range.setUpdateDeltaStartTimestamp( CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaStartTimestamp())); range.setLatestUpdateTimestamp( CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaEndTimestamp())); } return range; } } return range; }
/** * Returns the invalid timestamp range of a segment. * @return */ public List<UpdateVO> getInvalidTimestampRange() { List<UpdateVO> ranges = new ArrayList<UpdateVO>(); for (LoadMetadataDetails segment : segmentDetails) { if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus() || SegmentStatus.COMPACTED == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) { UpdateVO range = new UpdateVO(); range.setSegmentId(segment.getLoadName()); range.setFactTimestamp(segment.getLoadStartTime()); if (!segment.getUpdateDeltaStartTimestamp().isEmpty() && !segment.getUpdateDeltaEndTimestamp().isEmpty()) { range.setUpdateDeltaStartTimestamp( CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaStartTimestamp())); range.setLatestUpdateTimestamp( CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaEndTimestamp())); } ranges.add(range); } } return ranges; }
/** * Get the partition specs of the segment * @param segmentId * @param tablePath * @return * @throws IOException */ public static List<PartitionSpec> getPartitionSpecs(String segmentId, String tablePath) throws IOException { LoadMetadataDetails segEntry = null; LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath)); for (LoadMetadataDetails entry : details) { if (entry.getLoadName().equals(segmentId)) { segEntry = entry; break; } } if (segEntry != null && segEntry.getSegmentFile() != null) { SegmentFileStore fileStore = new SegmentFileStore(tablePath, segEntry.getSegmentFile()); List<PartitionSpec> partitionSpecs = fileStore.getPartitionSpecs(); for (PartitionSpec spec : partitionSpecs) { spec.setUuid(segmentId + "_" + segEntry.getLoadStartTime()); } return partitionSpecs; } return null; }