/** BEGIN - COMPACTION RELATED META-DATA MANAGEMENT **/ public Optional<byte[]> getInstantAuxiliaryDetails(HoodieInstant instant) { Path detailPath = new Path(metaClient.getMetaAuxiliaryPath(), instant.getFileName()); return readDataFromPath(detailPath); }
private void createFileInAuxiliaryFolder(HoodieInstant instant, Optional<byte[]> data) { Path fullPath = new Path(metaClient.getMetaAuxiliaryPath(), instant.getFileName()); createFileInPath(fullPath, data); }
/** * Remove older instants from auxiliary meta folder * * @param thresholdInstant Hoodie Instant * @return success if all eligible file deleted successfully * @throws IOException in case of error */ private boolean deleteAllInstantsOlderorEqualsInAuxMetaFolder(HoodieInstant thresholdInstant) throws IOException { List<HoodieInstant> instants = HoodieTableMetaClient.scanHoodieInstantsFromFileSystem(metaClient.getFs(), new Path(metaClient.getMetaAuxiliaryPath()), HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE); List<HoodieInstant> instantsToBeDeleted = instants.stream().filter(instant1 -> HoodieTimeline.compareTimestamps(instant1.getTimestamp(), thresholdInstant.getTimestamp(), LESSER_OR_EQUAL)).collect(Collectors.toList()); boolean success = true; for (HoodieInstant deleteInstant : instantsToBeDeleted) { log.info("Deleting instant " + deleteInstant + " in auxiliary meta path " + metaClient.getMetaAuxiliaryPath()); Path metaFile = new Path(metaClient.getMetaAuxiliaryPath(), deleteInstant.getFileName()); if (metaClient.getFs().exists(metaFile)) { success &= metaClient.getFs().delete(metaFile, false); log.info("Deleted instant file in auxiliary metapath : " + metaFile); } } return success; }
/** * Remove older instants from auxiliary meta folder * * @param thresholdInstant Hoodie Instant * @return success if all eligible file deleted successfully * @throws IOException in case of error */ private boolean deleteAllInstantsOlderorEqualsInAuxMetaFolder(HoodieInstant thresholdInstant) throws IOException { List<HoodieInstant> instants = HoodieTableMetaClient.scanHoodieInstantsFromFileSystem(metaClient.getFs(), new Path(metaClient.getMetaAuxiliaryPath()), HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE); List<HoodieInstant> instantsToBeDeleted = instants.stream().filter(instant1 -> HoodieTimeline.compareTimestamps(instant1.getTimestamp(), thresholdInstant.getTimestamp(), LESSER_OR_EQUAL)).collect(Collectors.toList()); boolean success = true; for (HoodieInstant deleteInstant : instantsToBeDeleted) { log.info("Deleting instant " + deleteInstant + " in auxiliary meta path " + metaClient.getMetaAuxiliaryPath()); Path metaFile = new Path(metaClient.getMetaAuxiliaryPath(), deleteInstant.getFileName()); if (metaClient.getFs().exists(metaFile)) { success &= metaClient.getFs().delete(metaFile, false); log.info("Deleted instant file in auxiliary metapath : " + metaFile); } } return success; }
new Path(metaClient.getMetaAuxiliaryPath()), HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE); assertEquals("Should delete all compaction instants < 104", 4, instants.size());
new Path(metaClient.getMetaAuxiliaryPath()), HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE); assertEquals("Should not delete any aux compaction files when maxCommitsToKeep is 5", 8, instants.size());