private void prepareLoadMetadata() { int loadCount = 0; Map<String, List<String>> snapshotMap = this.readCommittedIndexFileSnapShot.getSegmentIndexFileMap(); LoadMetadataDetails[] loadMetadataDetailsArray = new LoadMetadataDetails[snapshotMap.size()]; String segmentID; for (Map.Entry<String, List<String>> entry : snapshotMap.entrySet()) { segmentID = entry.getKey(); LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails(); long timeSet; try { timeSet = Long.parseLong(segmentID); } catch (NumberFormatException nu) { timeSet = 0; } loadMetadataDetails.setLoadEndTime(timeSet); loadMetadataDetails.setLoadStartTime(timeSet); loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS); loadMetadataDetails.setLoadName(segmentID); loadMetadataDetailsArray[loadCount++] = loadMetadataDetails; } this.loadMetadataDetails = loadMetadataDetailsArray; }
/** * Returns the invalid timestamp range of a segment. * @return */ public List<UpdateVO> getInvalidTimestampRange() { List<UpdateVO> ranges = new ArrayList<UpdateVO>(); for (LoadMetadataDetails segment : segmentDetails) { if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus() || SegmentStatus.COMPACTED == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) { UpdateVO range = new UpdateVO(); range.setSegmentId(segment.getLoadName()); range.setFactTimestamp(segment.getLoadStartTime()); if (!segment.getUpdateDeltaStartTimestamp().isEmpty() && !segment.getUpdateDeltaEndTimestamp().isEmpty()) { range.setUpdateDeltaStartTimestamp( CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaStartTimestamp())); range.setLatestUpdateTimestamp( CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaEndTimestamp())); } ranges.add(range); } } return ranges; }
if (SegmentStatus.SUCCESS == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus() || SegmentStatus.LOAD_PARTIAL_SUCCESS == segment.getSegmentStatus() || SegmentStatus.STREAMING == segment.getSegmentStatus() || SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) { if (null != segment.getMergedLoadName()) { Segment seg = new Segment(segment.getMergedLoadName(), segment.getSegmentFile(), readCommittedScope, segment); if (!listOfValidSegments.contains(seg)) { if (SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()) { listOfValidUpdatedSegments.add(seg); if (SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()) { new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope)); if (SegmentStatus.STREAMING == segment.getSegmentStatus() || SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) { listOfStreamSegments.add( new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope)); continue; new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope, segment)); } else if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus() || SegmentStatus.COMPACTED == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) {
/** * updates segment status and modificaton time details * * @param loadMetadata */ public static void updateSegmentMetadataDetails(LoadMetadataDetails loadMetadata) { // update status only if the segment is not marked for delete if (SegmentStatus.MARKED_FOR_DELETE != loadMetadata.getSegmentStatus()) { loadMetadata.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); loadMetadata.setModificationOrdeletionTimesStamp(CarbonUpdateUtil.readCurrentTime()); } }
private static boolean checkIfLoadCanBeDeletedPhysically(LoadMetadataDetails oneLoad, boolean isForceDelete) { if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() || SegmentStatus.COMPACTED == oneLoad.getSegmentStatus())) { if (isForceDelete) { return true; } long deletionTime = oneLoad.getModificationOrdeletionTimesStamp(); return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime); } return false; }
/** * This API will return the update status file name. * @param segmentList * @return */ public String getUpdateStatusFileName(LoadMetadataDetails[] segmentList) { if (segmentList.length == 0) { return ""; } else { for (LoadMetadataDetails eachSeg : segmentList) { // file name stored in 0th segment. if (eachSeg.getLoadName().equalsIgnoreCase("0")) { return eachSeg.getUpdateStatusFileName(); } } } return ""; }
if (loadDetail.getSegmentStatus() == SegmentStatus.MARKED_FOR_DELETE) { LOGGER.error("Compaction is aborted as the segment " + loadDetail.getLoadName() + " is deleted after the compaction is started."); return false; loadDetail.setSegmentStatus(SegmentStatus.COMPACTED); loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp); loadDetail.setMergedLoadName(mergedLoadNumber); LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails(); loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID); loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS); long loadEnddate = CarbonUpdateUtil.readCurrentTime(); loadMetadataDetails.setLoadEndTime(loadEnddate); CarbonTable carbonTable = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(); loadMetadataDetails.setLoadName(mergedLoadNumber); loadMetadataDetails.setSegmentFile(segmentFile); CarbonLoaderUtil .addDataIndexSizeIntoMetaEntry(loadMetadataDetails, mergedLoadNumber, carbonTable); loadMetadataDetails.setLoadStartTime(carbonLoadModel.getFactTimeStamp()); loadMetadataDetails.setPartitionCount("0"); loadMetadataDetails.setMajorCompacted("true");
LoadMetadataDetails streamSegment = null; for (LoadMetadataDetails detail : details) { if (FileFormat.ROW_V1 == detail.getFileFormat()) { if (SegmentStatus.STREAMING == detail.getSegmentStatus()) { streamSegment = detail; break; LoadMetadataDetails newDetail = new LoadMetadataDetails(); newDetail.setPartitionCount("0"); newDetail.setLoadName("" + segmentId); newDetail.setFileFormat(FileFormat.ROW_V1); newDetail.setLoadStartTime(System.currentTimeMillis()); newDetail.setSegmentStatus(SegmentStatus.STREAMING); SegmentStatusManager.writeLoadDetailsIntoFile( CarbonTablePath.getTableStatusFilePath(table.getTablePath()), newDetails); return newDetail.getLoadName(); } else { return streamSegment.getLoadName();
CarbonTablePath.getMetadataPath(table.getTablePath())); for (LoadMetadataDetails detail : details) { if (segmentId.equals(detail.getLoadName())) { detail.setLoadEndTime(System.currentTimeMillis()); detail.setSegmentStatus(SegmentStatus.STREAMING_FINISH); break; LoadMetadataDetails newDetail = new LoadMetadataDetails(); newDetail.setPartitionCount("0"); newDetail.setLoadName("" + newSegmentId); newDetail.setFileFormat(FileFormat.ROW_V1); newDetail.setLoadStartTime(System.currentTimeMillis()); newDetail.setSegmentStatus(SegmentStatus.STREAMING); .writeLoadDetailsIntoFile(CarbonTablePath.getTableStatusFilePath( table.getTablePath()), newDetails); return newDetail.getLoadName(); } else { LOGGER.error(
newMetaEntry.setLoadName(loadModel.getSegmentId()); } else { newMetaEntry.setLoadName(segmentId); loadModel.setSegmentId(segmentId); if (entry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS && SegmentStatusManager.isLoadInProgress( identifier, entry.getLoadName())) { throw new RuntimeException("Already insert overwrite is in progress"); } else if (newMetaEntry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS && entry.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS && SegmentStatusManager.isLoadInProgress( identifier, entry.getLoadName())) { throw new RuntimeException("Already insert into or load is in progress"); newMetaEntry.setLoadName(String.valueOf(loadModel.getSegmentId())); boolean found = false; for (LoadMetadataDetails entry : listOfLoadFolderDetails) { if (entry.getLoadName().equals(newMetaEntry.getLoadName()) && entry.getLoadStartTime() == newMetaEntry.getLoadStartTime()) { found = true; break; if (entry.getSegmentStatus() != SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); if (newMetaEntry.getSegmentStatus() == SegmentStatus.MARKED_FOR_DELETE) { addToStaleFolders(identifier, staleFolders, newMetaEntry);
if (loadMetadata.getLoadName().equalsIgnoreCase("0")) { loadMetadata.setUpdateStatusFileName( CarbonUpdateUtil.getUpdateStatusFileName(updatedTimeStamp)); if (segmentsToBeDeleted.contains(new Segment(loadMetadata.getLoadName()))) { loadMetadata.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); loadMetadata.setModificationOrdeletionTimesStamp(Long.parseLong(updatedTimeStamp)); if (loadMetadata.getLoadName().equalsIgnoreCase(segName.getSegmentNo())) { loadMetadata.setIsDeleted(CarbonCommonConstants.KEYWORD_TRUE); if (loadMetadata.getUpdateDeltaStartTimestamp().isEmpty()) { loadMetadata.setUpdateDeltaStartTimestamp(updatedTimeStamp); loadMetadata.setUpdateDeltaEndTimestamp(updatedTimeStamp); .contains(Segment.toSegment(loadMetadata.getLoadName(), null))) { loadMetadata.setSegmentFile(loadMetadata.getLoadName() + "_" + updatedTimeStamp + CarbonTablePath.SEGMENT_EXT);
if (loadId.equalsIgnoreCase(loadMetadata.getLoadName())) { segmentStatus = loadMetadata.getSegmentStatus(); if (SegmentStatus.COMPACTED == segmentStatus) { } else if (SegmentStatus.MARKED_FOR_DELETE != segmentStatus) { loadFound = true; loadMetadata.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); loadMetadata.setModificationOrdeletionTimesStamp(CarbonUpdateUtil.readCurrentTime()); LOG.info("Segment ID " + loadId + " Marked for Delete");
SegmentStatus loadStatus = loadMetadataDetail.getSegmentStatus(); if (loadStatus == SegmentStatus.SUCCESS || loadStatus == SegmentStatus.LOAD_PARTIAL_SUCCESS) { String dsize = loadMetadataDetail.getDataSize(); String isize = loadMetadataDetail.getIndexSize(); LOGGER.debug("It is an old segment, need calculate data size and index size again"); HashMap<String, Long> map = CarbonUtil.getDataSizeAndIndexSize( identifier.getTablePath(), loadMetadataDetail.getLoadName()); dsize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE)); isize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE)); loadMetadataDetail.setDataSize(dsize); loadMetadataDetail.setIndexSize(isize);
/** * Method to create new entry for load in table status file * * @param loadMetadataDetails * @param loadStatus * @param loadStartTime * @param addLoadEndTime */ public static void populateNewLoadMetaEntry(LoadMetadataDetails loadMetadataDetails, SegmentStatus loadStatus, long loadStartTime, boolean addLoadEndTime) { if (addLoadEndTime) { long loadEndDate = CarbonUpdateUtil.readCurrentTime(); loadMetadataDetails.setLoadEndTime(loadEndDate); } loadMetadataDetails.setSegmentStatus(loadStatus); loadMetadataDetails.setLoadStartTime(loadStartTime); }
.getUpdateDeltaFiles(loadsToMerge.get(0).getLoadName()); } catch (Exception e) { LOGGER.error("Error while getting the Update Delta Blocks."); loadDetail.setUpdateDeltaStartTimestamp(timestamp); loadDetail.setUpdateDeltaEndTimestamp(timestamp); if (loadDetail.getLoadName().equalsIgnoreCase("0")) { loadDetail .setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp)); .indexOf(Segment.toSegment(loadDetail.getLoadName(), null)); if (segmentFileIndex > -1) { loadDetail.setSegmentFile( segmentFilesToBeUpdated.get(segmentFileIndex).getSegmentFileName());
if (segment.getSegmentStatus() == SegmentStatus.STREAMING || segment.getSegmentStatus() == SegmentStatus.STREAMING_FINISH) { continue; String segId = segment.getLoadName(); if (segment.getSegmentFile() != null) { if (!StringUtils.isEmpty(segment.getDataSize())) { sizeOfOneSegmentAcrossPartition = Long.parseLong(segment.getDataSize()); } else { sizeOfOneSegmentAcrossPartition = CarbonUtil.getSizeOfSegment(carbonTable.getTablePath(), new Segment(segId, segment.getSegmentFile()));
/** * Return true if insert overwrite is in progress for specified table */ public static Boolean isOverwriteInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
SegmentStatus segmentStatus = null; for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) { Integer result = compareDateValues(loadMetadata.getLoadStartTimeAsLong(), loadStartTime); if (result < 0) { segmentStatus = loadMetadata.getSegmentStatus(); if (SegmentStatus.COMPACTED == segmentStatus) { LOG.info("Ignoring the segment : " + loadMetadata.getLoadName() + "as the segment has been compacted."); } else if (SegmentStatus.STREAMING == segmentStatus) { LOG.info("Ignoring the segment : " + loadMetadata.getLoadName() + "as the segment is streaming in progress."); } else if (SegmentStatus.INSERT_IN_PROGRESS == segmentStatus && isLoadInProgress( absoluteTableIdentifier, loadMetadata.getLoadName())) { LOG.info("Ignoring the segment : " + loadMetadata.getLoadName() + "as the segment is insert in progress."); } else if (SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segmentStatus && isLoadInProgress(absoluteTableIdentifier, loadMetadata.getLoadName())) { LOG.info("Ignoring the segment : " + loadMetadata.getLoadName() + "as the segment is insert overwrite in progress."); } else if (SegmentStatus.MARKED_FOR_DELETE != segmentStatus) { loadFound = true; updateSegmentMetadataDetails(loadMetadata); LOG.info("Info: " + loadStartTimeString + loadMetadata.getLoadStartTime() + " Marked for Delete");
boolean updated = false; for (LoadMetadataDetails detail : details) { if (SegmentStatus.STREAMING == detail.getSegmentStatus()) { detail.setLoadEndTime(System.currentTimeMillis()); detail.setSegmentStatus(SegmentStatus.STREAMING_FINISH); updated = true;
@Override public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) { double seg1Id = Double.parseDouble(seg1.getLoadName()); double seg2Id = Double.parseDouble(seg2.getLoadName()); return Double.compare(seg1Id, seg2Id); } });