private static LoadMetadataDetails getCurrentLoadStatusOfSegment(String segmentId, String metadataPath) { LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath); for (LoadMetadataDetails oneLoad : currentDetails) { if (oneLoad.getLoadName().equalsIgnoreCase(segmentId)) { return oneLoad; } } return null; }
/** * Read segments metadata from table status file and set it to this load model object */ public void readAndSetLoadMetadataDetails() { String metadataPath = CarbonTablePath.getMetadataPath(tablePath); LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metadataPath); setLoadMetadataDetails(Arrays.asList(details)); }
/** * Read the table status and get the segment corresponding to segmentNo * @param segmentNo * @param tablePath * @return */ public static Segment getSegment(String segmentNo, String tablePath) { LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath)); return getSegment(segmentNo, loadMetadataDetails); }
public static void physicalFactAndMeasureMetadataDeletion(CarbonTable carbonTable, LoadMetadataDetails[] newAddedLoadHistoryList, boolean isForceDelete, List<PartitionSpec> specs) { LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath()); physicalFactAndMeasureMetadataDeletion(carbonTable, currentDetails, isForceDelete, specs, currentDetails); if (newAddedLoadHistoryList != null && newAddedLoadHistoryList.length > 0) { physicalFactAndMeasureMetadataDeletion(carbonTable, newAddedLoadHistoryList, isForceDelete, specs, currentDetails); } }
long factTimeStamp = 0; LoadMetadataDetails[] segmentDetails = SegmentStatusManager.readLoadMetadata( CarbonTablePath.getMetadataPath(identifier.getTablePath()));
private static boolean isLoadDeletionRequired(String metaDataLocation) { LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation); if (details != null && details.length > 0) { for (LoadMetadataDetails oneRow : details) { if ((SegmentStatus.MARKED_FOR_DELETE == oneRow.getSegmentStatus() || SegmentStatus.COMPACTED == oneRow.getSegmentStatus() || SegmentStatus.INSERT_IN_PROGRESS == oneRow.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneRow.getSegmentStatus()) && oneRow.getVisibility().equalsIgnoreCase("true")) { return true; } } } return false; }
private static ReturnTuple isUpdationRequired( boolean isForceDeletion, CarbonTable carbonTable, AbsoluteTableIdentifier absoluteTableIdentifier) { LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath()); // Delete marked loads boolean isUpdationRequired = DeleteLoadFolders.deleteLoadFoldersFromFileSystem( absoluteTableIdentifier, isForceDeletion, details, carbonTable.getMetadataPath() ); return new ReturnTuple(details, isUpdationRequired); }
HashMap<String, Integer> uniqueSegments = new HashMap<>(); LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager.readLoadMetadata( CarbonTablePath.getMetadataPath(table.getTablePath())); for (CarbonInputSplit split : mbSplit.getAllSplits()) {
boolean forceDelete) throws IOException { LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(table.getMetadataPath());
listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath); if (listOfLoadFolderDetailsArray.length != 0) { updateDeletionStatus(identifier, loadDate, listOfLoadFolderDetailsArray, LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath); updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails);
listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath); if (listOfLoadFolderDetailsArray.length != 0) { updateDeletionStatus(identifier, loadIds, listOfLoadFolderDetailsArray, invalidLoadIds); LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath); updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails);
/** * Return true if insert overwrite is in progress for specified table */ public static Boolean isOverwriteInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
/** * Return true if any load or insert overwrite is in progress for specified table */ public static Boolean isLoadInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_IN_PROGRESS || segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
/** * Get the partition specs of the segment * @param segmentId * @param tablePath * @return * @throws IOException */ public static List<PartitionSpec> getPartitionSpecs(String segmentId, String tablePath) throws IOException { LoadMetadataDetails segEntry = null; LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath)); for (LoadMetadataDetails entry : details) { if (entry.getLoadName().equals(segmentId)) { segEntry = entry; break; } } if (segEntry != null && segEntry.getSegmentFile() != null) { SegmentFileStore fileStore = new SegmentFileStore(tablePath, segEntry.getSegmentFile()); List<PartitionSpec> partitionSpecs = fileStore.getPartitionSpecs(); for (PartitionSpec spec : partitionSpecs) { spec.setUuid(segmentId + "_" + segEntry.getLoadStartTime()); } return partitionSpecs; } return null; }
if (statusLock.lockWithRetries()) { LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath()); boolean updated = false; for (LoadMetadataDetails detail : details) {
public SegmentUpdateStatusManager(CarbonTable table) { this.identifier = table.getAbsoluteTableIdentifier(); this.isStandardTable = CarbonUtil.isStandardCarbonTable(table); // current it is used only for read function scenarios, as file update always requires to work // on latest file status. if (!table.getTableInfo().isTransactionalTable()) { // fileExist is costly operation, so check based on table Type segmentDetails = new LoadMetadataDetails[0]; } else { segmentDetails = SegmentStatusManager.readLoadMetadata( CarbonTablePath.getMetadataPath(identifier.getTablePath())); } if (segmentDetails.length != 0) { updateDetails = readLoadMetadata(); } else { updateDetails = new SegmentUpdateDetails[0]; } populateMap(); }
LOGGER.info("Acquired lock for tablepath" + tablePath + " for table status updation"); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metadataPath);
final boolean isCompactionFlow) throws IOException { String metaDataLocation = carbonTable.getMetadataPath(); final LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
SegmentStatusManager.readLoadMetadata( CarbonTablePath.getMetadataPath(table.getTablePath())); for (LoadMetadataDetails detail : details) {
SegmentStatusManager.readLoadMetadata( CarbonTablePath.getMetadataPath(table.getTablePath())); LoadMetadataDetails streamSegment = null;