private Stream<HoodieDataFile> convertFileStatusesToDataFiles(FileStatus[] statuses) { Predicate<FileStatus> roFilePredicate = fileStatus -> fileStatus.getPath().getName() .contains(metaClient.getTableConfig().getROFileFormat().getFileExtension()); return Arrays.stream(statuses).filter(roFilePredicate).map(HoodieDataFile::new); }
private Stream<Pair<String, String>> convertPathToFileIdWithCommitTime( final HoodieTableMetaClient metaClient, List<String> paths) { Predicate<String> roFilePredicate = path -> path.contains(metaClient.getTableConfig().getROFileFormat().getFileExtension()); Predicate<String> rtFilePredicate = path -> path.contains(metaClient.getTableConfig().getRTFileFormat().getFileExtension()); Stream<Pair<String, String>> stream1 = paths.stream().filter(roFilePredicate) .map(fullPath -> { String fileName = Paths.get(fullPath).getFileName().toString(); return Pair.of(FSUtils.getFileId(fileName), FSUtils.getCommitTime(fileName)); }); Stream<Pair<String, String>> stream2 = paths.stream().filter(rtFilePredicate) .map(path -> { return Pair.of(FSUtils.getFileIdFromLogPath(new Path(path)), FSUtils.getBaseCommitTimeFromLogPath(new Path(path))); }); return Stream.concat(stream1, stream2); } }
.filter(s -> s.contains((metaClient.getTableConfig().getROFileFormat().getFileExtension()))) .findAny() .map(f -> Pair.of(f, HoodieFileFormat.PARQUET)).orElseThrow(() -> {