private static SegmentStatusManager.ValidAndInvalidSegmentsInfo getValidAndInvalidSegments( CarbonTable carbonTable, Configuration configuration) throws IOException { SegmentStatusManager ssm = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier(), configuration); return ssm.getValidAndInvalidSegments(); }
private static LoadMetadataDetails getCurrentLoadStatusOfSegment(String segmentId, String metadataPath) { LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath); for (LoadMetadataDetails oneLoad : currentDetails) { if (oneLoad.getLoadName().equalsIgnoreCase(segmentId)) { return oneLoad; } } return null; }
/** * Return true if insert overwrite is in progress for specified table */ public static Boolean isOverwriteInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
SegmentStatus segmentStatus = null; for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) { Integer result = compareDateValues(loadMetadata.getLoadStartTimeAsLong(), loadStartTime); if (result < 0) { segmentStatus = loadMetadata.getSegmentStatus(); LOG.info("Ignoring the segment : " + loadMetadata.getLoadName() + "as the segment is streaming in progress."); } else if (SegmentStatus.INSERT_IN_PROGRESS == segmentStatus && isLoadInProgress( absoluteTableIdentifier, loadMetadata.getLoadName())) { LOG.info("Ignoring the segment : " + loadMetadata.getLoadName() + "as the segment is insert in progress."); } else if (SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segmentStatus && isLoadInProgress(absoluteTableIdentifier, loadMetadata.getLoadName())) { LOG.info("Ignoring the segment : " + loadMetadata.getLoadName() + "as the segment is insert overwrite in progress."); } else if (SegmentStatus.MARKED_FOR_DELETE != segmentStatus) { loadFound = true; updateSegmentMetadataDetails(loadMetadata); LOG.info("Info: " + loadStartTimeString + loadMetadata.getLoadStartTime() + " Marked for Delete");
AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(tablePath, null, null, tableId); SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier); ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock(); int retryCount = CarbonLockUtil .getLockProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK, LOGGER.info("Acquired lock for tablepath" + tablePath + " for table status updation"); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metadataPath); .writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
if (isLoadDeletionRequired(carbonTable.getMetadataPath())) { AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier(); boolean updationCompletionStatus = false; LoadMetadataDetails[] newAddedLoadHistoryList = null; ReturnTuple tuple = isUpdationRequired(isForceDeletion, carbonTable, identifier); if (tuple.isUpdateRequired) { ICarbonLock carbonTableStatusLock = ReturnTuple tuple2 = isUpdationRequired(isForceDeletion, carbonTable, identifier); if (!tuple2.isUpdateRequired) { return; SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath()); int maxSegmentId = SegmentStatusManager.getMaxSegmentId(tuple2.details); int invisibleSegmentCnt = SegmentStatusManager.countInvisibleSegments( tuple2.details, maxSegmentId); TableStatusReturnTuple tableStatusReturn = separateVisibleAndInvisibleSegments( tuple2.details, latestMetadata, invisibleSegmentCnt, maxSegmentId); LoadMetadataDetails[] oldLoadHistoryList = readLoadHistoryMetadata( carbonTable.getMetadataPath()); LoadMetadataDetails[] newLoadHistoryList = appendLoadHistoryList( oldLoadHistoryList, tableStatusReturn.arrayOfLoadHistoryDetails); writeLoadDetailsIntoFile( CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath()), tableStatusReturn.arrayOfLoadDetails); writeLoadDetailsIntoFile( CarbonTablePath.getTableStatusHistoryFilePath(carbonTable.getTablePath()),
listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath); if (listOfLoadFolderDetailsArray.length != 0) { updateDeletionStatus(identifier, loadDate, listOfLoadFolderDetailsArray, invalidLoadTimestamps, loadStartTime); if (invalidLoadTimestamps.isEmpty()) { LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath); updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails); writeLoadDetailsIntoFile(dataLoadLocation, listOfLoadFolderDetailsArray);
SegmentStatusManager ssm = new SegmentStatusManager(table.getAbsoluteTableIdentifier()); SegmentStatusManager.readLoadMetadata(table.getMetadataPath()); validUpdateStatusFile = ssm.getUpdateStatusFileName(details);
if (statusLock.lockWithRetries()) { LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath()); boolean updated = false; for (LoadMetadataDetails detail : details) { SegmentStatusManager.writeLoadDetailsIntoFile( CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath()), details);
public ValidAndInvalidSegmentsInfo getValidAndInvalidSegments() throws IOException { return getValidAndInvalidSegments(null, null); }
return invalidLoadIds; } else if (SegmentStatus.INSERT_IN_PROGRESS == segmentStatus && isLoadInProgress(absoluteTableIdentifier, loadId)) { return invalidLoadIds; } else if (SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segmentStatus && isLoadInProgress(absoluteTableIdentifier, loadId)) {
CarbonCommonConstants.DEFAULT_ENABLE_CALCULATE_SIZE); if (isCalculated.equalsIgnoreCase("true")) { SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier); ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock(); try { boolean lockAcquired = true; String metadataPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager.readLoadMetadata(metadataPath); SegmentStatusManager.writeLoadDetailsIntoFile( CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()), loadMetadataDetails);
listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath); if (listOfLoadFolderDetailsArray.length != 0) { updateDeletionStatus(identifier, loadIds, listOfLoadFolderDetailsArray, invalidLoadIds); if (invalidLoadIds.isEmpty()) { LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath); updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails); writeLoadDetailsIntoFile(dataLoadLocation, listOfLoadFolderDetailsArray);
throws IOException { SegmentStatusManager segmentStatusManager = new SegmentStatusManager(table.getAbsoluteTableIdentifier()); ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock(); try { if (carbonLock.lockWithRetries()) { SegmentStatusManager.readLoadMetadata( CarbonTablePath.getMetadataPath(table.getTablePath())); for (LoadMetadataDetails detail : details) { int newSegmentId = SegmentStatusManager.createNewSegmentId(details); LoadMetadataDetails newDetail = new LoadMetadataDetails(); newDetail.setPartitionCount("0"); .writeLoadDetailsIntoFile(CarbonTablePath.getTableStatusFilePath( table.getTablePath()), newDetails); return newDetail.getLoadName();
/** * This method returns the valid segments attached to the table Identifier. * * @param absoluteTableIdentifier * @return */ public static List<Segment> getValidSegmentList(AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegments = null; try { validAndInvalidSegments = new SegmentStatusManager(absoluteTableIdentifier).getValidAndInvalidSegments(); } catch (IOException e) { LOGGER.error("Error while getting valid segment list for a table identifier"); throw new IOException(); } return validAndInvalidSegments.getValidSegments(); }
/** * Read segments metadata from table status file and set it to this load model object */ public void readAndSetLoadMetadataDetails() { String metadataPath = CarbonTablePath.getMetadataPath(tablePath); LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metadataPath); setLoadMetadataDetails(Arrays.asList(details)); }
/** * Return true if any load or insert overwrite is in progress for specified table */ public static Boolean isLoadInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_IN_PROGRESS || segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
new SegmentStatusManager(table.getAbsoluteTableIdentifier()); ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock(); try { if (carbonLock.lockWithRetries()) { SegmentStatusManager.readLoadMetadata( CarbonTablePath.getMetadataPath(table.getTablePath())); LoadMetadataDetails streamSegment = null; int segmentId = SegmentStatusManager.createNewSegmentId(details); LoadMetadataDetails newDetail = new LoadMetadataDetails(); newDetail.setPartitionCount("0"); SegmentStatusManager.writeLoadDetailsIntoFile( CarbonTablePath.getTableStatusFilePath(table.getTablePath()), newDetails); return newDetail.getLoadName();
@Override public void deleteDatamapData() { SegmentStatusManager ssm = new SegmentStatusManager(getCarbonTable().getAbsoluteTableIdentifier()); try { List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments(); for (Segment segment : validSegments) { deleteDatamapData(segment); } } catch (IOException e) { LOGGER.error("drop datamap failed, failed to delete datamap directory"); } }
/** * Read the table status and get the segment corresponding to segmentNo * @param segmentNo * @param tablePath * @return */ public static Segment getSegment(String segmentNo, String tablePath) { LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath)); return getSegment(segmentNo, loadMetadataDetails); }