/** * Create a file system view, as of the given timeline, with the provided file statuses. */ public HoodieTableFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline, FileStatus[] fileStatuses) { this(metaClient, visibleActiveTimeline); addFilesToView(fileStatuses); }
@Override public Stream<HoodieDataFile> getAllDataFiles(String partitionPath) { return getAllFileGroups(partitionPath) .map(HoodieFileGroup::getAllDataFiles) .flatMap(dataFileList -> dataFileList) .filter(df -> !isDataFileDueToPendingCompaction(df)); }
@Override public Stream<FileSlice> getAllFileSlices(String partitionPath) { return getAllFileGroups(partitionPath) .map(HoodieFileGroup::getAllFileSlices) .flatMap(sliceList -> sliceList); }
/** * Get the view of the file system for this table */ public TableFileSystemView getFileSystemView() { return new HoodieTableFileSystemView(metaClient, getCompletedCommitsTimeline()); }
@Test public void testViewForFileSlicesWithNoBaseFileAndRequestedCompaction() throws Exception { testViewForFileSlicesWithAsyncCompaction(true, false); }
/** * With async compaction, it is possible to see partial/complete data-files due to inflight-compactions, * Ignore those data-files * @param fileSlice File Slice * @return */ private FileSlice filterDataFileAfterPendingCompaction(FileSlice fileSlice) { if (isFileSliceAfterPendingCompaction(fileSlice)) { // Data file is filtered out of the file-slice as the corresponding compaction // instant not completed yet. FileSlice transformed = new FileSlice(fileSlice.getBaseInstantTime(), fileSlice.getFileId()); fileSlice.getLogFiles().forEach(transformed::addLogFile); return transformed; } return fileSlice; }
/** * Get the read optimized view of the file system for this table */ public TableFileSystemView.ReadOptimizedView getROFileSystemView() { return new HoodieTableFileSystemView(metaClient, getCompletedCommitsTimeline()); }
@Override public Stream<FileSlice> getLatestFileSlices(String partitionPath) { return getAllFileGroups(partitionPath) .map(HoodieFileGroup::getLatestFileSlice) .filter(Optional::isPresent) .map(Optional::get) .map(this::filterDataFileAfterPendingCompaction); }
@Override public Stream<HoodieDataFile> getLatestDataFiles(final String partitionPath) { return getAllFileGroups(partitionPath) .map(fileGroup -> { return fileGroup.getAllDataFiles().filter(df -> !isDataFileDueToPendingCompaction(df)).findFirst(); }) .filter(Optional::isPresent) .map(Optional::get); }
@Test public void testViewForFileSlicesWithNoBaseFileAndInflightCompaction() throws Exception { testViewForFileSlicesWithAsyncCompaction(true, true); }
/** * Get the completed (commit + compaction) view of the file system for this table */ public TableFileSystemView getCompletedFileSystemView() { return new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline()); }
@Override public Stream<FileSlice> getLatestFileSlicesBeforeOrOn(String partitionPath, String maxCommitTime) { return getAllFileGroups(partitionPath) .map(fileGroup -> fileGroup.getLatestFileSliceBeforeOrOn(maxCommitTime)) .filter(Optional::isPresent) .map(Optional::get) .map(this::filterDataFileAfterPendingCompaction); }
@Test public void testViewForFileSlicesWithBaseFileAndRequestedCompaction() throws Exception { testViewForFileSlicesWithAsyncCompaction(false, false); }
/** * Get the view of the file system for this table */ public TableFileSystemView getFileSystemView() { return new HoodieTableFileSystemView(metaClient, getCompletedCommitTimeline()); }
/** * Returns all file-slices including uncommitted ones. * @param partitionPath * @return */ private Stream<FileSlice> getAllRawFileSlices(String partitionPath) { return fsView.getAllFileGroups(partitionPath) .map(group -> group.getAllFileSlicesIncludingInflight()) .flatMap(sliceList -> sliceList); }
@Test public void testViewForFileSlicesWithBaseFileAndInflightCompaction() throws Exception { testViewForFileSlicesWithAsyncCompaction(false, true); }
/** * Get the read optimized view of the file system for this table */ public TableFileSystemView.ReadOptimizedView getROFileSystemView() { return new HoodieTableFileSystemView(metaClient, getCompletedCommitTimeline()); }
/** * Get the completed (commit + compaction) view of the file system for this table */ public TableFileSystemView getCompletedFileSystemView() { return new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline()); }
/** * Get the real time view of the file system for this table */ public TableFileSystemView.RealtimeView getRTFileSystemView() { return new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline().filterCompletedAndCompactionInstants()); }
/** * Get the real time view of the file system for this table */ public TableFileSystemView.RealtimeView getRTFileSystemView() { return new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline().filterCompletedAndCompactionInstants()); }