public String getFileId() { return FSUtils.getFileIdFromLogPath(path); }
/** * Check if the file is a parquet file of a log file. Then get the fileId appropriately. */ public static String getFileIdFromFilePath(Path filePath) { if (FSUtils.isLogFile(filePath)) { return FSUtils.getFileIdFromLogPath(filePath); } return FSUtils.getFileId(filePath.getName()); }
public CompactionOperation(java.util.Optional<HoodieDataFile> dataFile, String partitionPath, List<HoodieLogFile> logFiles, Map<String, Double> metrics) { if (dataFile.isPresent()) { this.baseInstantTime = dataFile.get().getCommitTime(); this.dataFilePath = Optional.of(dataFile.get().getPath()); this.fileId = dataFile.get().getFileId(); this.dataFileCommitTime = Optional.of(dataFile.get().getCommitTime()); } else { assert logFiles.size() > 0; this.dataFilePath = Optional.absent(); this.baseInstantTime = FSUtils.getBaseCommitTimeFromLogPath(logFiles.get(0).getPath()); this.fileId = FSUtils.getFileIdFromLogPath(logFiles.get(0).getPath()); this.dataFileCommitTime = Optional.absent(); } this.partitionPath = partitionPath; this.deltaFilePaths = logFiles.stream().map(s -> s.getPath().toString()) .collect(Collectors.toList()); this.metrics = metrics; }
private Stream<Pair<String, String>> convertPathToFileIdWithCommitTime( final HoodieTableMetaClient metaClient, List<String> paths) { Predicate<String> roFilePredicate = path -> path.contains(metaClient.getTableConfig().getROFileFormat().getFileExtension()); Predicate<String> rtFilePredicate = path -> path.contains(metaClient.getTableConfig().getRTFileFormat().getFileExtension()); Stream<Pair<String, String>> stream1 = paths.stream().filter(roFilePredicate) .map(fullPath -> { String fileName = Paths.get(fullPath).getFileName().toString(); return Pair.of(FSUtils.getFileId(fileName), FSUtils.getCommitTime(fileName)); }); Stream<Pair<String, String>> stream2 = paths.stream().filter(rtFilePredicate) .map(path -> { return Pair.of(FSUtils.getFileIdFromLogPath(new Path(path)), FSUtils.getBaseCommitTimeFromLogPath(new Path(path))); }); return Stream.concat(stream1, stream2); } }