/** * Add a new log file into the group */ public void addLogFile(HoodieLogFile logFile) { if (!fileSlices.containsKey(logFile.getBaseCommitTime())) { fileSlices.put(logFile.getBaseCommitTime(), new FileSlice(logFile.getBaseCommitTime(), id)); } fileSlices.get(logFile.getBaseCommitTime()).addLogFile(logFile); }
/** * Get all the log files for the passed in FileId in the partition path */ public static Stream<HoodieLogFile> getAllLogFiles(FileSystem fs, Path partitionPath, final String fileId, final String logFileExtension, final String baseCommitTime) throws IOException { return Arrays.stream(fs.listStatus(partitionPath, path -> path.getName().startsWith("." + fileId) && path.getName() .contains(logFileExtension))) .map(HoodieLogFile::new).filter(s -> s.getBaseCommitTime().equals(baseCommitTime)); }
public static Comparator<HoodieLogFile> getBaseInstantAndLogVersionComparator() { return (o1, o2) -> { String baseInstantTime1 = o1.getBaseCommitTime(); String baseInstantTime2 = o2.getBaseCommitTime(); if (baseInstantTime1.equals(baseInstantTime2)) { // reverse the order by log-version when base-commit is same return new Integer(o2.getLogVersion()).compareTo(o1.getLogVersion()); } // reverse the order by base-commits return baseInstantTime2.compareTo(baseInstantTime1); }; }
public HoodieLogFile rollOver(FileSystem fs) throws IOException { String fileId = getFileId(); String baseCommitTime = getBaseCommitTime(); String extension = "." + FSUtils.getFileExtensionFromLog(path); int newVersion = FSUtils .computeNextLogVersion(fs, path.getParent(), fileId, extension, baseCommitTime); return new HoodieLogFile(new Path(path.getParent(), FSUtils.makeLogFileName(fileId, extension, baseCommitTime, newVersion))); }
log.error("Error renaming log file", e); log.error("\n\n\n***NOTE Compaction is in inconsistent state. Try running \"compaction repair " + lfPair.getLeft().getBaseCommitTime() + "\" to recover from failure ***\n\n\n"); return new RenameOpResult(lfPair, false, Optional.of(e));
logFilesInFileSlice.stream().filter(lf -> !logFilesInCompactionOp.contains(lf)) .collect(Collectors.toSet()); Preconditions.checkArgument(diff.stream().allMatch(lf -> lf.getBaseCommitTime().equals(compactionInstant)), "There are some log-files which are neither specified in compaction plan " + "nor present after compaction request instant. Some of these :" + diff);
.filter(fs -> fs.getFileId().equals(operation.getFileId())).findFirst().get(); List<HoodieLogFile> logFilesToRepair = merged.getLogFiles().filter(lf -> lf.getBaseCommitTime().equals(compactionInstant)) .sorted(HoodieLogFile.getBaseInstantAndLogVersionComparator().reversed()) .collect(Collectors.toList());
.mapToLong(lf -> lf.getFileSize().get()).sum(); long logFilesScheduledForCompactionTotalSize = fs.getLogFiles().filter(lf -> lf.getFileSize().isPresent()) .filter(lf -> lf.getBaseCommitTime().equals(fs.getBaseInstantTime())) .mapToLong(lf -> lf.getFileSize().get()).sum(); row[idx++] = logFilesScheduledForCompactionTotalSize; .filter(lf -> !lf.getBaseCommitTime().equals(fs.getBaseInstantTime())) .mapToLong(lf -> lf.getFileSize().get()).sum(); row[idx++] = logFilesUnscheduledTotalSize; .filter(lf -> lf.getBaseCommitTime().equals(fs.getBaseInstantTime())) .collect(Collectors.toList()).toString(); row[idx++] = fs.getLogFiles().filter(lf -> lf.getFileSize().isPresent()) .filter(lf -> !lf.getBaseCommitTime().equals(fs.getBaseInstantTime())) .collect(Collectors.toList()).toString();
private void validateRenameFiles(List<Pair<HoodieLogFile, HoodieLogFile>> renameFiles, String ingestionInstant, String compactionInstant, HoodieTableFileSystemView fsView) { // Ensure new names of log-files are on expected lines Set<HoodieLogFile> uniqNewLogFiles = new HashSet<>(); Set<HoodieLogFile> uniqOldLogFiles = new HashSet<>(); renameFiles.stream().forEach(lfPair -> { Assert.assertFalse("Old Log File Names do not collide", uniqOldLogFiles.contains(lfPair.getKey())); Assert.assertFalse("New Log File Names do not collide", uniqNewLogFiles.contains(lfPair.getValue())); uniqOldLogFiles.add(lfPair.getKey()); uniqNewLogFiles.add(lfPair.getValue()); }); renameFiles.stream().forEach(lfPair -> { HoodieLogFile oldLogFile = lfPair.getLeft(); HoodieLogFile newLogFile = lfPair.getValue(); Assert.assertEquals("Base Commit time is expected", ingestionInstant, newLogFile.getBaseCommitTime()); Assert.assertEquals("Base Commit time is expected", compactionInstant, oldLogFile.getBaseCommitTime()); Assert.assertEquals("File Id is expected", oldLogFile.getFileId(), newLogFile.getFileId()); HoodieLogFile lastLogFileBeforeCompaction = fsView.getLatestMergedFileSlicesBeforeOrOn(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0], ingestionInstant) .filter(fs -> fs.getFileId().equals(oldLogFile.getFileId())) .map(fs -> fs.getLogFiles().findFirst().get()).findFirst().get(); Assert.assertEquals("Log Version expected", lastLogFileBeforeCompaction.getLogVersion() + oldLogFile.getLogVersion(), newLogFile.getLogVersion()); Assert.assertTrue("Log version does not collide", newLogFile.getLogVersion() > lastLogFileBeforeCompaction.getLogVersion()); }); }