TableSegmentRefresher(CarbonTable table) { SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(table); SegmentUpdateDetails[] updateStatusDetails = statusManager.getUpdateStatusDetails(); for (SegmentUpdateDetails updateDetails : updateStatusDetails) { UpdateVO updateVO = statusManager.getInvalidTimestampRange(updateDetails.getSegmentName()); segmentRefreshTime.put(updateVO.getSegmentId(), new SegmentRefreshInfo(updateVO.getCreatedOrUpdatedTimeStamp(), 0)); } }
Boolean isIUDTable = false; isIUDTable = (updateStatusManager.getUpdateStatusDetails().length != 0); updateStatusManager.getInvalidTimestampRange(inputSplit.getSegmentId()); .getDeleteDeltaFilePath(inputSplit.getPath().toString(), inputSplit.getSegmentId()); } catch (Exception e) { throw new IOException(e);
/** * Return row count of input block */ public static long getRowCount( BlockMappingVO blockMappingVO, CarbonTable carbonTable) { SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(carbonTable); long rowCount = 0; Map<String, Long> blockRowCountMap = blockMappingVO.getBlockRowCountMapping(); for (Map.Entry<String, Long> blockRowEntry : blockRowCountMap.entrySet()) { String key = blockRowEntry.getKey(); long alreadyDeletedCount = 0; SegmentUpdateDetails detail = updateStatusManager.getDetailsForABlock(key); if (detail != null) { alreadyDeletedCount = Long.parseLong(detail.getDeletedRowsInBlock()); } rowCount += (blockRowEntry.getValue() - alreadyDeletedCount); } return rowCount; }
public SegmentUpdateStatusManager(CarbonTable table, LoadMetadataDetails[] segmentDetails) { this.identifier = table.getAbsoluteTableIdentifier(); this.isStandardTable = CarbonUtil.isStandardCarbonTable(table); // current it is used only for read function scenarios, as file update always requires to work // on latest file status. this.segmentDetails = segmentDetails; updateDetails = readLoadMetadata(); populateMap(); }
CarbonTable table, String updateStatusFileIdentifier, boolean isCompaction) { boolean status = false; SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table); ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock(); boolean lockStatus = false; .getUpdateStatusDetails(); segmentUpdateStatusManager.writeLoadDetailsIntoFile(oldList, updateStatusFileIdentifier); status = true; } else {
new SegmentUpdateStatusManager(carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable()); ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock(); ICarbonLock statusLock = segmentStatusManager.getTableStatusLock(); .getUpdateDeltaFiles(loadsToMerge.get(0).getLoadName()); } catch (Exception e) { LOGGER.error("Error while getting the Update Delta Blocks."); .readLoadMetadata(); segmentUpdateStatusManager.writeLoadDetailsIntoFile( Arrays.asList(updateLists), timestamp); SegmentStatusManager.writeLoadDetailsIntoFile(
LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList(); SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager( table, loadMetadataDetails); SegmentStatusManager.ValidAndInvalidSegmentsInfo allSegments = .getTableSegmentRefresher(getOrCreateCarbonTable(job.getConfiguration())) .isRefreshNeeded(eachSegment, updateStatusManager.getInvalidTimestampRange(eachSegment.getSegmentNo())); if (refreshNeeded) { toBeCleanedSegments.add(eachSegment); SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key);
SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(table); .getUpdateDeltaFilesList(segment.getLoadName(), false, CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles, isInvalidFile); .getUpdateDeltaFilesList(segment.getLoadName(), false, CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles, isInvalidFile); SegmentUpdateDetails[] updateDetails = updateStatusManager.readLoadMetadata(); for (SegmentUpdateDetails block : updateDetails) { CarbonFile[] completeListOfDeleteDeltaFiles; .getDeleteDeltaInvalidFilesList(block, false, allSegmentFiles, isAbortedFile); for (CarbonFile invalidFile : invalidDeleteDeltaFiles) { .getDeleteDeltaInvalidFilesList(block, true, allSegmentFiles, isInvalidFile); for (CarbonFile invalidFile : completeListOfDeleteDeltaFiles) { .getDeleteDeltaInvalidFilesList(block, false, allSegmentFiles, isInvalidFile); for (CarbonFile invalidFile : invalidDeleteDeltaFiles) {
Long timestamp) throws IOException { SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table); segmentUpdateStatusManager.setUpdateStatusDetails(segmentUpdateDetails); segmentUpdateStatusManager.getDeleteDeltaFilesList(new Segment(seg), blockName);
new SegmentUpdateStatusManager(carbonTable, loadMetadataDetails); List<Segment> invalidSegments = new ArrayList<>(); List<UpdateVO> invalidTimestampsList = new ArrayList<>(); invalidTimestampsList.addAll(updateStatusManager.getInvalidTimestampRange()); if (invalidSegments.size() > 0) { DataMapStoreManager.getInstance()
partitionInfo, oldPartitionIdList, new SegmentUpdateStatusManager(carbonTable));
for (SegmentUpdateDetails blockDetails : getUpdateStatusDetails()) { if (blockDetails.getActualBlockName().equalsIgnoreCase(eachFile.getName()) && CarbonUpdateUtil.isBlockInvalid(blockDetails.getSegmentStatus())) {
/** * check the block whether is valid * * @param segName segment name * @param blockName block name * @return the status of block whether is valid */ public boolean isBlockValid(String segName, String blockName) { SegmentUpdateDetails details = getDetailsForABlock(segName, blockName); return details == null || !CarbonUpdateUtil.isBlockInvalid(details.getSegmentStatus()); } /**
String tableUpdateStatusIdentifier = getUpdatedStatusIdentifier(); return new SegmentUpdateDetails[0]; } finally { closeStreams(buffReader, inStream, dataInputStream);
DataMapStoreManager.getInstance().getTableSegmentRefresher(carbonTable) .isRefreshNeeded(filteredSegment, updateStatusManager.getInvalidTimestampRange(filteredSegment.getSegmentNo())); if (refreshNeeded) { toBeCleanedSegments.add(filteredSegment);
for (String file : entry.getValue()) { String[] deltaFilePaths = updateStatusManager.getDeleteDeltaFilePath(file, segment.getSegmentNo()); for (String deltaFilePath : deltaFilePaths) { FileFactory.deleteFile(deltaFilePath, FileFactory.getFileType(deltaFilePath));
public SegmentUpdateStatusManager(CarbonTable table) { this.identifier = table.getAbsoluteTableIdentifier(); this.isStandardTable = CarbonUtil.isStandardCarbonTable(table); // current it is used only for read function scenarios, as file update always requires to work // on latest file status. if (!table.getTableInfo().isTransactionalTable()) { // fileExist is costly operation, so check based on table Type segmentDetails = new LoadMetadataDetails[0]; } else { segmentDetails = SegmentStatusManager.readLoadMetadata( CarbonTablePath.getMetadataPath(identifier.getTablePath())); } if (segmentDetails.length != 0) { updateDetails = readLoadMetadata(); } else { updateDetails = new SegmentUpdateDetails[0]; } populateMap(); }
new SegmentUpdateStatusManager(carbonTable, currLoadDetails); for (final LoadMetadataDetails oneLoad : loadDetails) { if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
.getUpdateStatusDetails()) { if (origDetails.getBlockName().equalsIgnoreCase(carbonDataMergerUtilResult.getBlockName()) && origDetails.getSegmentName()
/** * * @param blockMappingVO * @param segmentUpdateStatusManager */ public static void createBlockDetailsMap(BlockMappingVO blockMappingVO, SegmentUpdateStatusManager segmentUpdateStatusManager) { Map<String, Long> blockRowCountMap = blockMappingVO.getBlockRowCountMapping(); Map<String, RowCountDetailsVO> outputMap = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); for (Map.Entry<String, Long> blockRowEntry : blockRowCountMap.entrySet()) { String key = blockRowEntry.getKey(); long alreadyDeletedCount = 0; SegmentUpdateDetails detail = segmentUpdateStatusManager.getDetailsForABlock(key); if (null != detail) { alreadyDeletedCount = Long.parseLong(detail.getDeletedRowsInBlock()); } RowCountDetailsVO rowCountDetailsVO = new RowCountDetailsVO(blockRowEntry.getValue(), alreadyDeletedCount); outputMap.put(key, rowCountDetailsVO); } blockMappingVO.setCompleteBlockRowDetailVO(outputMap); }