public Builder withCleanerPolicy(HoodieCleaningPolicy policy) { props.setProperty(CLEANER_POLICY_PROP, policy.name()); return this; }
/** * compaction properties **/ public HoodieCleaningPolicy getCleanerPolicy() { return HoodieCleaningPolicy .valueOf(props.getProperty(HoodieCompactionConfig.CLEANER_POLICY_PROP)); }
/** * compaction properties **/ public HoodieCleaningPolicy getCleanerPolicy() { return HoodieCleaningPolicy .valueOf(props.getProperty(HoodieCompactionConfig.CLEANER_POLICY_PROP)); }
public Builder withCleanerPolicy(HoodieCleaningPolicy policy) { props.setProperty(CLEANER_POLICY_PROP, policy.name()); return this; }
TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP, DEFAULT_TARGET_PARTITIONS_PER_DAYBASED_COMPACTION); HoodieCleaningPolicy.valueOf(props.getProperty(CLEANER_POLICY_PROP)); Preconditions.checkArgument(Integer.parseInt(props.getProperty(MAX_COMMITS_TO_KEEP)) > Integer .parseInt(props.getProperty(MIN_COMMITS_TO_KEEP)));
/** * Returns files to be cleaned for the given partitionPath based on cleaning policy. */ public List<String> getDeletePaths(String partitionPath) throws IOException { HoodieCleaningPolicy policy = config.getCleanerPolicy(); List<String> deletePaths; if (policy == HoodieCleaningPolicy.KEEP_LATEST_COMMITS) { deletePaths = getFilesToCleanKeepingLatestCommits(partitionPath); } else if (policy == HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) { deletePaths = getFilesToCleanKeepingLatestVersions(partitionPath); } else { throw new IllegalArgumentException("Unknown cleaning policy : " + policy.name()); } logger.info(deletePaths.size() + " patterns used to delete in partition path:" + partitionPath); return deletePaths; }
TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP, DEFAULT_TARGET_PARTITIONS_PER_DAYBASED_COMPACTION); HoodieCleaningPolicy.valueOf(props.getProperty(CLEANER_POLICY_PROP)); Preconditions.checkArgument(Integer.parseInt(props.getProperty(MAX_COMMITS_TO_KEEP)) > Integer .parseInt(props.getProperty(MIN_COMMITS_TO_KEEP)));
/** * Returns files to be cleaned for the given partitionPath based on cleaning policy. */ public List<String> getDeletePaths(String partitionPath) throws IOException { HoodieCleaningPolicy policy = config.getCleanerPolicy(); List<String> deletePaths; if (policy == HoodieCleaningPolicy.KEEP_LATEST_COMMITS) { deletePaths = getFilesToCleanKeepingLatestCommits(partitionPath); } else if (policy == HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS) { deletePaths = getFilesToCleanKeepingLatestVersions(partitionPath); } else { throw new IllegalArgumentException("Unknown cleaning policy : " + policy.name()); } logger.info(deletePaths.size() + " patterns used to delete in partition path:" + partitionPath); return deletePaths; }
.valueOf(getProperty(HOODIE_CLEANER_POLICY, DEFAULT_HOODIE_CLEANER_POLICY))); compactionConfigBuilder.retainCommits( getProperty(HOODIE_CLEANER_COMMITS_RETAINED, DEFAULT_HOODIE_CLEANER_COMMITS_RETAINED));
public static HoodieCleanMetadata convertCleanMetadata(String startCleanTime, Optional<Long> durationInMs, List<HoodieCleanStat> cleanStats) { ImmutableMap.Builder<String, HoodieCleanPartitionMetadata> partitionMetadataBuilder = ImmutableMap.builder(); int totalDeleted = 0; String earliestCommitToRetain = null; for (HoodieCleanStat stat : cleanStats) { HoodieCleanPartitionMetadata metadata = new HoodieCleanPartitionMetadata(stat.getPartitionPath(), stat.getPolicy().name(), stat.getDeletePathPatterns(), stat.getSuccessDeleteFiles(), stat.getDeletePathPatterns()); partitionMetadataBuilder.put(stat.getPartitionPath(), metadata); totalDeleted += stat.getSuccessDeleteFiles().size(); if (earliestCommitToRetain == null) { // This will be the same for all partitions earliestCommitToRetain = stat.getEarliestCommitToRetain(); } } return new HoodieCleanMetadata(startCleanTime, durationInMs.orElseGet(() -> -1L), totalDeleted, earliestCommitToRetain, partitionMetadataBuilder.build()); }