if (blockDetail.getDeleteDeltaStartTimestamp().isEmpty() || (isCompaction == true)) { blockDetail .setDeleteDeltaStartTimestamp(newBlockEntry.getDeleteDeltaStartTimestamp()); blockDetail.setDeleteDeltaEndTimestamp(newBlockEntry.getDeleteDeltaEndTimestamp()); blockDetail.setSegmentStatus(newBlockEntry.getSegmentStatus()); blockDetail.setDeletedRowsInBlock(newBlockEntry.getDeletedRowsInBlock()); } else {
/** * This method returns the list of Blocks associated with the segment * from the SegmentUpdateDetails List. * @param segmentName * @return */ public List<String> getBlockNameFromSegment(String segmentName) { List<String> blockNames = new ArrayList<String>(); for (SegmentUpdateDetails block : updateDetails) { if (block.getSegmentName().equalsIgnoreCase(segmentName) && !CarbonUpdateUtil .isBlockInvalid(block.getSegmentStatus())) { blockNames.add(block.getBlockName()); } } return blockNames; }
/** * populate the block and its details in a map. */ private void populateMap() { blockAndDetailsMap = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); for (SegmentUpdateDetails blockDetails : updateDetails) { String blockIdentifier = CarbonUpdateUtil .getSegmentBlockNameKey(blockDetails.getSegmentName(), blockDetails.getActualBlockName()); blockAndDetailsMap.put(blockIdentifier, blockDetails); } }
SegmentUpdateDetails tempSegmentUpdateDetails = new SegmentUpdateDetails(); tempSegmentUpdateDetails.setSegmentName(carbonDataMergerUtilResult.getSegmentName()); tempSegmentUpdateDetails.setBlockName(carbonDataMergerUtilResult.getBlockName()); if (origDetails.getBlockName().equalsIgnoreCase(carbonDataMergerUtilResult.getBlockName()) && origDetails.getSegmentName() .equalsIgnoreCase(carbonDataMergerUtilResult.getSegmentName())) { tempSegmentUpdateDetails.setDeletedRowsInBlock(origDetails.getDeletedRowsInBlock()); tempSegmentUpdateDetails.setSegmentStatus(origDetails.getSegmentStatus()); break; tempSegmentUpdateDetails.setDeleteDeltaStartTimestamp( carbonDataMergerUtilResult.getDeleteDeltaStartTimestamp()); tempSegmentUpdateDetails .setDeleteDeltaEndTimestamp(carbonDataMergerUtilResult.getDeleteDeltaEndTimestamp());
if (updateLists[i].getBlockName().equalsIgnoreCase(compactedBlocks) && updateLists[i].getSegmentStatus() != SegmentStatus.COMPACTED && updateLists[i].getSegmentStatus() != SegmentStatus.MARKED_FOR_DELETE) { updateLists[i].setSegmentStatus(SegmentStatus.COMPACTED);
CarbonFile[] invalidDeleteDeltaFiles; if (!block.getSegmentName().equalsIgnoreCase(segment.getLoadName())) { continue; if (CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) { completeListOfDeleteDeltaFiles = updateStatusManager .getDeleteDeltaInvalidFilesList(block, true,
if (blockDetails.getActualBlockName().equalsIgnoreCase(eachFile.getName()) && CarbonUpdateUtil.isBlockInvalid(blockDetails.getSegmentStatus())) { validBlock = false;
/** * check the block whether is valid * * @param segName segment name * @param blockName block name * @return the status of block whether is valid */ public boolean isBlockValid(String segName, String blockName) { SegmentUpdateDetails details = getDetailsForABlock(segName, blockName); return details == null || !CarbonUpdateUtil.isBlockInvalid(details.getSegmentStatus()); } /**
/** * * @param details * @param segmentBlockCount */ public static void decrementDeletedBlockCount(SegmentUpdateDetails details, Map<String, Long> segmentBlockCount) { String segId = details.getSegmentName(); segmentBlockCount.put(details.getSegmentName(), segmentBlockCount.get(segId) - 1); }
/** * Return row count of input block */ public static long getRowCount( BlockMappingVO blockMappingVO, CarbonTable carbonTable) { SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(carbonTable); long rowCount = 0; Map<String, Long> blockRowCountMap = blockMappingVO.getBlockRowCountMapping(); for (Map.Entry<String, Long> blockRowEntry : blockRowCountMap.entrySet()) { String key = blockRowEntry.getKey(); long alreadyDeletedCount = 0; SegmentUpdateDetails detail = updateStatusManager.getDetailsForABlock(key); if (detail != null) { alreadyDeletedCount = Long.parseLong(detail.getDeletedRowsInBlock()); } rowCount += (blockRowEntry.getValue() - alreadyDeletedCount); } return rowCount; }
if (needCompleteList && block.getBlockName().equalsIgnoreCase(blkName)) { files.add(eachFile); CarbonTablePath.DataFileUtil.getTimeStampFromDeleteDeltaFile(fileName)); if (block.getBlockName().equalsIgnoreCase(blkName)) {
if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getSegmentStatus())) { Long blockCount = blockRowCountMapping.get(key); if (blockCount == null) {
TableSegmentRefresher(CarbonTable table) { SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(table); SegmentUpdateDetails[] updateStatusDetails = statusManager.getUpdateStatusDetails(); for (SegmentUpdateDetails updateDetails : updateStatusDetails) { UpdateVO updateVO = statusManager.getInvalidTimestampRange(updateDetails.getSegmentName()); segmentRefreshTime.put(updateVO.getSegmentId(), new SegmentRefreshInfo(updateVO.getCreatedOrUpdatedTimeStamp(), 0)); } }
/** * * @param blockMappingVO * @param segmentUpdateStatusManager */ public static void createBlockDetailsMap(BlockMappingVO blockMappingVO, SegmentUpdateStatusManager segmentUpdateStatusManager) { Map<String, Long> blockRowCountMap = blockMappingVO.getBlockRowCountMapping(); Map<String, RowCountDetailsVO> outputMap = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); for (Map.Entry<String, Long> blockRowEntry : blockRowCountMap.entrySet()) { String key = blockRowEntry.getKey(); long alreadyDeletedCount = 0; SegmentUpdateDetails detail = segmentUpdateStatusManager.getDetailsForABlock(key); if (null != detail) { alreadyDeletedCount = Long.parseLong(detail.getDeletedRowsInBlock()); } RowCountDetailsVO rowCountDetailsVO = new RowCountDetailsVO(blockRowEntry.getValue(), alreadyDeletedCount); outputMap.put(key, rowCountDetailsVO); } blockMappingVO.setCompleteBlockRowDetailVO(outputMap); }
/** * Returns all delta file paths of specified block * * @param blockDir block directory with CarbonFile format * @param blockNameFromTuple block name from tuple * @param extension the file extension name * @param segment the segment name * @return the list of delete file */ private List<String> getDeltaFiles(CarbonFile blockDir, final String blockNameFromTuple, final String extension, String segment) throws IOException { List<String> deleteFileList = new ArrayList<>(); for (SegmentUpdateDetails block : updateDetails) { if (block.getBlockName().equalsIgnoreCase(blockNameFromTuple) && block.getSegmentName() .equalsIgnoreCase(segment) && !CarbonUpdateUtil .isBlockInvalid(block.getSegmentStatus())) { final long deltaStartTimestamp = getStartTimeOfDeltaFile(extension, block); // If there is no delete delete file , then return null if (deltaStartTimestamp == 0) { return deleteFileList; } final long deltaEndTimeStamp = getEndTimeOfDeltaFile(extension, block); // final long deltaEndTimeStamp = block.getDeleteDeltaEndTimeAsLong(); // final long deltaStartTimestamp = block.getDeleteDeltaStartTimeAsLong(); return getFilePaths(blockDir, blockNameFromTuple, extension, deleteFileList, deltaStartTimestamp, deltaEndTimeStamp); } } return deleteFileList; }
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath)); for (SegmentUpdateDetails block : updateDetails) { if ((block.getBlockName().equalsIgnoreCase(blockName)) && (block.getSegmentName().equalsIgnoreCase(segmentId.getSegmentNo())) && !CarbonUpdateUtil.isBlockInvalid((block.getSegmentStatus()))) { final long deltaStartTimestamp = getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);