private Stream<HoodieLogFile> convertFileStatusesToLogFiles(FileStatus[] statuses) { Predicate<FileStatus> rtFilePredicate = fileStatus -> fileStatus.getPath().getName() .contains(metaClient.getTableConfig().getRTFileFormat().getFileExtension()); return Arrays.stream(statuses).filter(rtFilePredicate).map(HoodieLogFile::new); }
/** * Get the Read Optimized Storage Format * * @return HoodieFileFormat for the Read Optimized Storage format */ public HoodieFileFormat getRTFileFormat() { if (props.containsKey(HOODIE_RT_FILE_FORMAT_PROP_NAME)) { return HoodieFileFormat.valueOf(props.getProperty(HOODIE_RT_FILE_FORMAT_PROP_NAME)); } return DEFAULT_RT_FILE_FORMAT; }
private Stream<HoodieDataFile> convertFileStatusesToDataFiles(FileStatus[] statuses) { Predicate<FileStatus> roFilePredicate = fileStatus -> fileStatus.getPath().getName() .contains(metaClient.getTableConfig().getROFileFormat().getFileExtension()); return Arrays.stream(statuses).filter(roFilePredicate).map(HoodieDataFile::new); }
/** * Get the Read Optimized Storage Format * * @return HoodieFileFormat for the Read Optimized Storage format */ public HoodieFileFormat getROFileFormat() { if (props.containsKey(HOODIE_RO_FILE_FORMAT_PROP_NAME)) { return HoodieFileFormat.valueOf(props.getProperty(HOODIE_RO_FILE_FORMAT_PROP_NAME)); } return DEFAULT_RO_FILE_FORMAT; }
private Stream<Pair<String, String>> convertPathToFileIdWithCommitTime( final HoodieTableMetaClient metaClient, List<String> paths) { Predicate<String> roFilePredicate = path -> path.contains(metaClient.getTableConfig().getROFileFormat().getFileExtension()); Predicate<String> rtFilePredicate = path -> path.contains(metaClient.getTableConfig().getRTFileFormat().getFileExtension()); Stream<Pair<String, String>> stream1 = paths.stream().filter(roFilePredicate) .map(fullPath -> { String fileName = Paths.get(fullPath).getFileName().toString(); return Pair.of(FSUtils.getFileId(fileName), FSUtils.getCommitTime(fileName)); }); Stream<Pair<String, String>> stream2 = paths.stream().filter(rtFilePredicate) .map(path -> { return Pair.of(FSUtils.getFileIdFromLogPath(new Path(path)), FSUtils.getBaseCommitTimeFromLogPath(new Path(path))); }); return Stream.concat(stream1, stream2); } }
.filter(s -> s.contains((metaClient.getTableConfig().getROFileFormat().getFileExtension()))) .findAny() .map(f -> Pair.of(f, HoodieFileFormat.PARQUET)).orElseThrow(() -> {