@Override public Stream<FileSlice> getAllFileSlices(String partitionPath) { return getAllFileGroups(partitionPath) .map(HoodieFileGroup::getAllFileSlices) .flatMap(sliceList -> sliceList); }
@Override public Stream<FileSlice> getLatestFileSlices(String partitionPath) { return getAllFileGroups(partitionPath) .map(HoodieFileGroup::getLatestFileSlice) .filter(Optional::isPresent) .map(Optional::get) .map(this::filterDataFileAfterPendingCompaction); }
@Override public Stream<HoodieDataFile> getAllDataFiles(String partitionPath) { return getAllFileGroups(partitionPath) .map(HoodieFileGroup::getAllDataFiles) .flatMap(dataFileList -> dataFileList) .filter(df -> !isDataFileDueToPendingCompaction(df)); }
@Override public Stream<FileSlice> getLatestFileSlicesBeforeOrOn(String partitionPath, String maxCommitTime) { return getAllFileGroups(partitionPath) .map(fileGroup -> fileGroup.getLatestFileSliceBeforeOrOn(maxCommitTime)) .filter(Optional::isPresent) .map(Optional::get) .map(this::filterDataFileAfterPendingCompaction); }
@Override public Stream<HoodieDataFile> getLatestDataFiles(final String partitionPath) { return getAllFileGroups(partitionPath) .map(fileGroup -> { return fileGroup.getAllDataFiles().filter(df -> !isDataFileDueToPendingCompaction(df)).findFirst(); }) .filter(Optional::isPresent) .map(Optional::get); }
@Override public Stream<FileSlice> getLatestMergedFileSlicesBeforeOrOn(String partitionPath, String maxInstantTime) { return getAllFileGroups(partitionPath) .map(fileGroup -> { Optional<FileSlice> fileSlice = fileGroup.getLatestFileSliceBeforeOrOn(maxInstantTime); // if the file-group is under construction, pick the latest before compaction instant time. if (fileSlice.isPresent()) { fileSlice = Optional.of(getMergedFileSlice(fileGroup, fileSlice.get())); } return fileSlice; }) .filter(Optional::isPresent) .map(Optional::get); }
@Override public Stream<HoodieDataFile> getLatestDataFilesOn(String partitionPath, String instantTime) { return getAllFileGroups(partitionPath) .map(fileGroup -> fileGroup.getAllDataFiles() .filter(dataFile -> HoodieTimeline.compareTimestamps(dataFile.getCommitTime(), instantTime, HoodieTimeline.EQUAL)) .filter(df -> !isDataFileDueToPendingCompaction(df)) .findFirst()) .filter(Optional::isPresent) .map(Optional::get); }
@Override public Stream<HoodieDataFile> getLatestDataFilesBeforeOrOn(String partitionPath, String maxCommitTime) { return getAllFileGroups(partitionPath) .map(fileGroup -> fileGroup.getAllDataFiles() .filter(dataFile -> HoodieTimeline.compareTimestamps(dataFile.getCommitTime(), maxCommitTime, HoodieTimeline.LESSER_OR_EQUAL)) .filter(df -> !isDataFileDueToPendingCompaction(df)) .findFirst()) .filter(Optional::isPresent) .map(Optional::get); }
@Override public Stream<FileSlice> getLatestUnCompactedFileSlices(String partitionPath) { return getAllFileGroups(partitionPath) .map(fileGroup -> { FileSlice fileSlice = fileGroup.getLatestFileSlice().get(); // if the file-group is under compaction, pick the latest before compaction instant time. if (isFileSliceAfterPendingCompaction(fileSlice)) { String compactionInstantTime = fileIdToPendingCompaction.get(fileSlice.getFileId()).getLeft(); return fileGroup.getLatestFileSliceBefore(compactionInstantTime); } return Optional.of(fileSlice); }) .map(Optional::get); }
/** * Returns latest raw file-slices including uncommitted ones. * @param partitionPath * @return */ public Stream<FileSlice> getLatestRawFileSlices(String partitionPath) { return fsView.getAllFileGroups(partitionPath) .map(fileGroup -> fileGroup.getLatestFileSlicesIncludingInflight()) .filter(fileSliceOpt -> fileSliceOpt.isPresent()) .map(Optional::get); }
/** * Returns all file-slices including uncommitted ones. * @param partitionPath * @return */ private Stream<FileSlice> getAllRawFileSlices(String partitionPath) { return fsView.getAllFileGroups(partitionPath) .map(group -> group.getAllFileSlicesIncludingInflight()) .flatMap(sliceList -> sliceList); }
includeInflight, excludeCompaction); List<Comparable[]> rows = new ArrayList<>(); fsView.getAllFileGroups().forEach(fg -> fg.getAllFileSlices().forEach(fs -> { int idx = 0;
List<HoodieFileGroup> fileGroups = fsView.getAllFileGroups("2016/05/01").collect(Collectors.toList()); assertEquals(3, fileGroups.size());
List<HoodieFileGroup> fileGroups = fsView.getAllFileGroups(partitionPath).collect(Collectors.toList()); assertEquals(3, fileGroups.size()); for (HoodieFileGroup fileGroup : fileGroups) {
newFsView.getAllFileGroups(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0]).flatMap(fg -> fg.getAllFileSlices()) .filter(fs -> fs.getBaseInstantTime().equals(ingestionInstant)) .map(fs -> Pair.of(fs.getFileId(), fs.getLogFiles().count()))
newFsView.getAllFileGroups(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0]).flatMap(fg -> fg.getAllFileSlices()) .filter(fs -> fs.getBaseInstantTime().equals(ingestionInstant)) .filter(fs -> fs.getFileId().equals(op.getFileId()))