public static HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, String compactionInstant) throws IOException { HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan( metaClient.getActiveTimeline().getInstantAuxiliaryDetails( HoodieTimeline.getCompactionRequestedInstant(compactionInstant)).get()); return compactionPlan; }
/** * Construction Compaction Plan from compaction instant */ private static HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, String compactionInstant) throws IOException { HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan( metaClient.getActiveTimeline().getInstantAuxiliaryDetails( HoodieTimeline.getCompactionRequestedInstant(compactionInstant)).get()); return compactionPlan; }
/** * Commit a compaction operation. Allow passing additional meta-data to be stored in commit instant file. */ public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus> writeStatuses, Optional<Map<String, String>> extraMetadata) throws IOException { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true); HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc); HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan( timeline.getInstantAuxiliaryDetails(HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime)).get()); // Merge extra meta-data passed by user with the one already in inflight compaction Optional<Map<String, String>> mergedMetaData = extraMetadata.map(m -> { Map<String, String> merged = new HashMap<>(); Map<String, String> extraMetaDataFromInstantFile = compactionPlan.getExtraMetadata(); if (extraMetaDataFromInstantFile != null) { merged.putAll(extraMetaDataFromInstantFile); } // Overwrite/Merge with the user-passed meta-data merged.putAll(m); return Optional.of(merged); }).orElseGet(() -> Optional.ofNullable(compactionPlan.getExtraMetadata())); commitCompaction(writeStatuses, table, compactionInstantTime, true, mergedMetaData); }
/** * Commit a compaction operation. Allow passing additional meta-data to be stored in commit instant file. */ public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus> writeStatuses, Optional<Map<String, String>> extraMetadata) throws IOException { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true); HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc); HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan( timeline.getInstantAuxiliaryDetails(HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime)).get()); // Merge extra meta-data passed by user with the one already in inflight compaction Optional<Map<String, String>> mergedMetaData = extraMetadata.map(m -> { Map<String, String> merged = new HashMap<>(); Map<String, String> extraMetaDataFromInstantFile = compactionPlan.getExtraMetadata(); if (extraMetaDataFromInstantFile != null) { merged.putAll(extraMetaDataFromInstantFile); } // Overwrite/Merge with the user-passed meta-data merged.putAll(m); return Optional.of(merged); }).orElseGet(() -> Optional.ofNullable(compactionPlan.getExtraMetadata())); commitCompaction(writeStatuses, table, compactionInstantTime, true, mergedMetaData); }
HoodieTimeline.getCompactionRequestedInstant(instant.getTimestamp())).get()); } catch (HoodieIOException ioe) { HoodieTimeline.getCompactionRequestedInstant(instant.getTimestamp())).get());
private void moveCompactionFromRequestedToInflight(String compactionInstantTime, HoodieWriteClient client, HoodieWriteConfig cfg) throws IOException { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath()); HoodieInstant compactionInstant = HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime); HoodieCompactionPlan workload = AvroUtils.deserializeCompactionPlan( metaClient.getActiveTimeline().getInstantAuxiliaryDetails(compactionInstant).get()); metaClient.getActiveTimeline().transitionCompactionRequestedToInflight(compactionInstant); HoodieInstant instant = metaClient.getActiveTimeline().reload().filterPendingCompactionTimeline().getInstants() .filter(in -> in.getTimestamp().equals(compactionInstantTime)).findAny().get(); assertTrue("Instant must be marked inflight", instant.isInflight()); }
HoodieCompactionPlan workload = AvroUtils.deserializeCompactionPlan( activeTimeline.getInstantAuxiliaryDetails( HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime)).get());
/** * Ensures compaction instant is in expected state and performs Compaction for the workload stored in instant-time * @param compactionInstantTime Compaction Instant Time * @return * @throws IOException */ private JavaRDD<WriteStatus> compact(String compactionInstantTime, boolean autoCommit) throws IOException { // Create a Hoodie table which encapsulated the commits and files visible HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true); HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc); HoodieTimeline pendingCompactionTimeline = metaClient.getActiveTimeline().filterPendingCompactionTimeline(); HoodieInstant inflightInstant = HoodieTimeline.getCompactionInflightInstant(compactionInstantTime); if (pendingCompactionTimeline.containsInstant(inflightInstant)) { //inflight compaction - Needs to rollback first deleting new parquet files before we run compaction. rollbackInflightCompaction(inflightInstant, table); // refresh table metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true); table = HoodieTable.getHoodieTable(metaClient, config, jsc); pendingCompactionTimeline = metaClient.getActiveTimeline().filterPendingCompactionTimeline(); } HoodieInstant instant = HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime); if (pendingCompactionTimeline.containsInstant(instant)) { return runCompaction(instant, metaClient.getActiveTimeline(), autoCommit); } else { throw new IllegalStateException("No Compaction request available at " + compactionInstantTime + " to run compaction"); } }
HoodieInstant requested = HoodieTimeline.getCompactionRequestedInstant(compactionInstant.getTimestamp()); commitTimeline.saveToCompactionRequested(requested, AvroUtils.serializeCompactionPlan(compactionPlan)); commitTimeline.transitionCompactionRequestedToInflight(requested);
/** * Ensures compaction instant is in expected state and performs Compaction for the workload stored in instant-time * @param compactionInstantTime Compaction Instant Time * @return * @throws IOException */ private JavaRDD<WriteStatus> compact(String compactionInstantTime, boolean autoCommit) throws IOException { // Create a Hoodie table which encapsulated the commits and files visible HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true); HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc); HoodieTimeline pendingCompactionTimeline = metaClient.getActiveTimeline().filterPendingCompactionTimeline(); HoodieInstant inflightInstant = HoodieTimeline.getCompactionInflightInstant(compactionInstantTime); if (pendingCompactionTimeline.containsInstant(inflightInstant)) { //inflight compaction - Needs to rollback first deleting new parquet files before we run compaction. rollbackInflightCompaction(inflightInstant, table); // refresh table metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true); table = HoodieTable.getHoodieTable(metaClient, config, jsc); pendingCompactionTimeline = metaClient.getActiveTimeline().filterPendingCompactionTimeline(); } HoodieInstant instant = HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime); if (pendingCompactionTimeline.containsInstant(instant)) { return runCompaction(instant, metaClient.getActiveTimeline(), autoCommit); } else { throw new IllegalStateException("No Compaction request available at " + compactionInstantTime + " to run compaction"); } }