public static void deleteOlderCleanMetaFiles(FileSystem fs, String metaPath, Stream<HoodieInstant> instants) { //TODO - this should be archived when archival is made general for all meta-data // skip MIN_CLEAN_TO_KEEP and delete rest instants.skip(MIN_CLEAN_TO_KEEP).map(s -> { try { return fs.delete(new Path(metaPath, s.getFileName()), false); } catch (IOException e) { throw new HoodieIOException("Could not delete clean meta files" + s.getFileName(), e); } }); }
public static void deleteOlderRollbackMetaFiles(FileSystem fs, String metaPath, Stream<HoodieInstant> instants) { //TODO - this should be archived when archival is made general for all meta-data // skip MIN_ROLLBACK_TO_KEEP and delete rest instants.skip(MIN_ROLLBACK_TO_KEEP).map(s -> { try { return fs.delete(new Path(metaPath, s.getFileName()), false); } catch (IOException e) { throw new HoodieIOException( "Could not delete rollback meta files " + s.getFileName(), e); } }); }
public void createInflight(HoodieInstant instant) { log.info("Creating a new in-flight instant " + instant); // Create the in-flight file createFileInMetaPath(instant.getFileName(), Optional.empty()); }
@Override public String apply(HoodieInstant hoodieInstant) { return hoodieInstant.getFileName(); } })).collect(Collectors.toList());
@Override public Optional<byte[]> getInstantDetails(HoodieInstant instant) { Path detailPath = new Path(metaClient.getMetaPath(), instant.getFileName()); return readDataFromPath(detailPath); }
/** BEGIN - COMPACTION RELATED META-DATA MANAGEMENT **/ public Optional<byte[]> getInstantAuxiliaryDetails(HoodieInstant instant) { Path detailPath = new Path(metaClient.getMetaAuxiliaryPath(), instant.getFileName()); return readDataFromPath(detailPath); }
private void createFileInAuxiliaryFolder(HoodieInstant instant, Optional<byte[]> data) { Path fullPath = new Path(metaClient.getMetaAuxiliaryPath(), instant.getFileName()); createFileInPath(fullPath, data); }
public void saveToInflight(HoodieInstant instant, Optional<byte[]> content) { Preconditions.checkArgument(instant.isInflight()); createFileInMetaPath(instant.getFileName(), content); }
public void saveToCompactionRequested(HoodieInstant instant, Optional<byte[]> content) { Preconditions.checkArgument(instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)); // Write workload to auxiliary folder createFileInAuxiliaryFolder(instant, content); createFileInMetaPath(instant.getFileName(), content); }
private void deleteInstantFile(HoodieInstant instant) { log.info("Deleting instant " + instant); Path inFlightCommitFilePath = new Path(metaClient.getMetaPath(), instant.getFileName()); try { boolean result = metaClient.getFs().delete(inFlightCommitFilePath, false); if (result) { log.info("Removed in-flight " + instant); } else { throw new HoodieIOException("Could not delete in-flight instant " + instant); } } catch (IOException e) { throw new HoodieIOException( "Could not remove inflight commit " + inFlightCommitFilePath, e); } }
private void revertCompleteToInflight(HoodieInstant completed, HoodieInstant inflight) { Preconditions.checkArgument(completed.getTimestamp().equals(inflight.getTimestamp())); Path inFlightCommitFilePath = new Path(metaClient.getMetaPath(), inflight.getFileName()); try { if (!metaClient.getFs().exists(inFlightCommitFilePath)) { Path commitFilePath = new Path(metaClient.getMetaPath(), completed.getFileName()); boolean success = metaClient.getFs().rename(commitFilePath, inFlightCommitFilePath); if (!success) { throw new HoodieIOException( "Could not rename " + commitFilePath + " to " + inFlightCommitFilePath); } } } catch (IOException e) { throw new HoodieIOException("Could not complete revert " + completed, e); } }
/** * END - COMPACTION RELATED META-DATA MANAGEMENT **/ private void transitionState(HoodieInstant fromInstant, HoodieInstant toInstant, Optional<byte[]> data) { Preconditions.checkArgument(fromInstant.getTimestamp().equals(toInstant.getTimestamp())); Path commitFilePath = new Path(metaClient.getMetaPath(), toInstant.getFileName()); try { // open a new file and write the commit metadata in Path inflightCommitFile = new Path(metaClient.getMetaPath(), fromInstant.getFileName()); createFileInMetaPath(fromInstant.getFileName(), data); boolean success = metaClient.getFs().rename(inflightCommitFile, commitFilePath); if (!success) { throw new HoodieIOException( "Could not rename " + inflightCommitFile + " to " + commitFilePath); } } catch (IOException e) { throw new HoodieIOException("Could not complete " + fromInstant, e); } }
private boolean deleteArchivedInstants(List<HoodieInstant> archivedInstants) throws IOException { log.info("Deleting instants " + archivedInstants); boolean success = true; for (HoodieInstant archivedInstant : archivedInstants) { Path commitFile = new Path(metaClient.getMetaPath(), archivedInstant.getFileName()); try { if (metaClient.getFs().exists(commitFile)) { success &= metaClient.getFs().delete(commitFile, false); log.info("Archived and deleted instant file " + commitFile); } } catch (IOException e) { throw new HoodieIOException("Failed to delete archived instant " + archivedInstant, e); } } // Remove older meta-data from auxiliary path too Optional<HoodieInstant> latestCommitted = archivedInstants.stream() .filter(i -> { return i.isCompleted() && (i.getAction().equals(COMMIT_ACTION) || (i.getAction().equals(DELTA_COMMIT_ACTION))); }) .sorted(Comparator.comparing(HoodieInstant::getTimestamp).reversed()).findFirst(); if (latestCommitted.isPresent()) { success &= deleteAllInstantsOlderorEqualsInAuxMetaFolder(latestCommitted.get()); } return success; }
private boolean deleteArchivedInstants(List<HoodieInstant> archivedInstants) throws IOException { log.info("Deleting instants " + archivedInstants); boolean success = true; for (HoodieInstant archivedInstant : archivedInstants) { Path commitFile = new Path(metaClient.getMetaPath(), archivedInstant.getFileName()); try { if (metaClient.getFs().exists(commitFile)) { success &= metaClient.getFs().delete(commitFile, false); log.info("Archived and deleted instant file " + commitFile); } } catch (IOException e) { throw new HoodieIOException("Failed to delete archived instant " + archivedInstant, e); } } // Remove older meta-data from auxiliary path too Optional<HoodieInstant> latestCommitted = archivedInstants.stream() .filter(i -> { return i.isCompleted() && (i.getAction().equals(COMMIT_ACTION) || (i.getAction().equals(DELTA_COMMIT_ACTION))); }).max(Comparator.comparing(HoodieInstant::getTimestamp)); if (latestCommitted.isPresent()) { success &= deleteAllInstantsOlderorEqualsInAuxMetaFolder(latestCommitted.get()); } return success; }
/** * Remove older instants from auxiliary meta folder * * @param thresholdInstant Hoodie Instant * @return success if all eligible file deleted successfully * @throws IOException in case of error */ private boolean deleteAllInstantsOlderorEqualsInAuxMetaFolder(HoodieInstant thresholdInstant) throws IOException { List<HoodieInstant> instants = HoodieTableMetaClient.scanHoodieInstantsFromFileSystem(metaClient.getFs(), new Path(metaClient.getMetaAuxiliaryPath()), HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE); List<HoodieInstant> instantsToBeDeleted = instants.stream().filter(instant1 -> HoodieTimeline.compareTimestamps(instant1.getTimestamp(), thresholdInstant.getTimestamp(), LESSER_OR_EQUAL)).collect(Collectors.toList()); boolean success = true; for (HoodieInstant deleteInstant : instantsToBeDeleted) { log.info("Deleting instant " + deleteInstant + " in auxiliary meta path " + metaClient.getMetaAuxiliaryPath()); Path metaFile = new Path(metaClient.getMetaAuxiliaryPath(), deleteInstant.getFileName()); if (metaClient.getFs().exists(metaFile)) { success &= metaClient.getFs().delete(metaFile, false); log.info("Deleted instant file in auxiliary metapath : " + metaFile); } } return success; }
/** * Remove older instants from auxiliary meta folder * * @param thresholdInstant Hoodie Instant * @return success if all eligible file deleted successfully * @throws IOException in case of error */ private boolean deleteAllInstantsOlderorEqualsInAuxMetaFolder(HoodieInstant thresholdInstant) throws IOException { List<HoodieInstant> instants = HoodieTableMetaClient.scanHoodieInstantsFromFileSystem(metaClient.getFs(), new Path(metaClient.getMetaAuxiliaryPath()), HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE); List<HoodieInstant> instantsToBeDeleted = instants.stream().filter(instant1 -> HoodieTimeline.compareTimestamps(instant1.getTimestamp(), thresholdInstant.getTimestamp(), LESSER_OR_EQUAL)).collect(Collectors.toList()); boolean success = true; for (HoodieInstant deleteInstant : instantsToBeDeleted) { log.info("Deleting instant " + deleteInstant + " in auxiliary meta path " + metaClient.getMetaAuxiliaryPath()); Path metaFile = new Path(metaClient.getMetaAuxiliaryPath(), deleteInstant.getFileName()); if (metaClient.getFs().exists(metaFile)) { success &= metaClient.getFs().delete(metaFile, false); log.info("Deleted instant file in auxiliary metapath : " + metaFile); } } return success; }
public static void createCompactionAuxiliaryMetadata(String basePath, HoodieInstant instant, Configuration configuration) throws IOException { Path commitFile = new Path( basePath + "/" + HoodieTableMetaClient.AUXILIARYFOLDER_NAME + "/" + instant.getFileName()); FileSystem fs = FSUtils.getFs(basePath, configuration); FSDataOutputStream os = fs.create(commitFile, true); HoodieCompactionPlan workload = new HoodieCompactionPlan(); try { // Write empty commit metadata os.writeBytes(new String(AvroUtils.serializeCompactionPlan(workload).get(), StandardCharsets.UTF_8)); } finally { os.close(); } }
.build(); HoodieInstant inflight = new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, compactionInstant); Path inflightPath = new Path(metaClient.getMetaPath(), inflight.getFileName()); if (metaClient.getFs().exists(inflightPath)) {
HoodieInstant inflight = new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, compactionOperationWithInstant.getLeft()); Path inflightPath = new Path(metaClient.getMetaPath(), inflight.getFileName()); if (metaClient.getFs().exists(inflightPath)) {