@Override public Stream<FileSlice> getLatestUnCompactedFileSlices(String partitionPath) { return getAllFileGroups(partitionPath) .map(fileGroup -> { FileSlice fileSlice = fileGroup.getLatestFileSlice().get(); // if the file-group is under compaction, pick the latest before compaction instant time. if (isFileSliceAfterPendingCompaction(fileSlice)) { String compactionInstantTime = fileIdToPendingCompaction.get(fileSlice.getFileId()).getLeft(); return fileGroup.getLatestFileSliceBefore(compactionInstantTime); } return Optional.of(fileSlice); }) .map(Optional::get); }
/** * If the file-slice is because of pending compaction instant, this method merges the file-slice with the one before * the compaction instant time * @param fileGroup File Group for which the file slice belongs to * @param fileSlice File Slice which needs to be merged * @return */ private FileSlice getMergedFileSlice(HoodieFileGroup fileGroup, FileSlice fileSlice) { // if the file-group is under construction, pick the latest before compaction instant time. if (fileIdToPendingCompaction.containsKey(fileSlice.getFileId())) { String compactionInstantTime = fileIdToPendingCompaction.get(fileSlice.getFileId()).getKey(); if (fileSlice.getBaseInstantTime().equals(compactionInstantTime)) { Optional<FileSlice> prevFileSlice = fileGroup.getLatestFileSliceBefore(compactionInstantTime); if (prevFileSlice.isPresent()) { return mergeCompactionPendingFileSlices(fileSlice, prevFileSlice.get()); } } } return fileSlice; }