private JavaRDD<WriteStatus> updateIndexAndCommitIfNeeded(JavaRDD<WriteStatus> writeStatusRDD, HoodieTable<T> table, String commitTime) { // Update the index back JavaRDD<WriteStatus> statuses = index.updateLocation(writeStatusRDD, jsc, table); // Trigger the insert and collect statuses statuses = statuses.persist(config.getWriteStatusStorageLevel()); commitOnAutoCommit(commitTime, statuses, new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true) .getCommitActionType()); return statuses; }
private JavaRDD<WriteStatus> updateIndexAndCommitIfNeeded(JavaRDD<WriteStatus> writeStatusRDD, HoodieTable<T> table, String commitTime) { // Update the index back JavaRDD<WriteStatus> statuses = index.updateLocation(writeStatusRDD, jsc, table); // Trigger the insert and collect statuses statuses = statuses.persist(config.getWriteStatusStorageLevel()); commitOnAutoCommit(commitTime, statuses, new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true) .getCommitActionType()); return statuses; }
/** * Perform compaction operations as specified in the compaction commit file * * @param compactionInstant Compacton Instant time * @param activeTimeline Active Timeline * @param autoCommit Commit after compaction * @return RDD of Write Status */ private JavaRDD<WriteStatus> runCompaction( HoodieInstant compactionInstant, HoodieActiveTimeline activeTimeline, boolean autoCommit) throws IOException { HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan( activeTimeline.getInstantAuxiliaryDetails(compactionInstant).get()); // Mark instant as compaction inflight activeTimeline.transitionCompactionRequestedToInflight(compactionInstant); compactionTimer = metrics.getCompactionCtx(); // Create a Hoodie table which encapsulated the commits and files visible HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true); HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc); JavaRDD<WriteStatus> statuses = table.compact(jsc, compactionInstant.getTimestamp(), compactionPlan); // Force compaction action statuses.persist(config.getWriteStatusStorageLevel()); // pass extra-metada so that it gets stored in commit file automatically commitCompaction(statuses, table, compactionInstant.getTimestamp(), autoCommit, Optional.ofNullable(compactionPlan.getExtraMetadata())); return statuses; }
/** * Perform compaction operations as specified in the compaction commit file * * @param compactionInstant Compacton Instant time * @param activeTimeline Active Timeline * @param autoCommit Commit after compaction * @return RDD of Write Status */ private JavaRDD<WriteStatus> runCompaction( HoodieInstant compactionInstant, HoodieActiveTimeline activeTimeline, boolean autoCommit) throws IOException { HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan( activeTimeline.getInstantAuxiliaryDetails(compactionInstant).get()); // Mark instant as compaction inflight activeTimeline.transitionCompactionRequestedToInflight(compactionInstant); compactionTimer = metrics.getCompactionCtx(); // Create a Hoodie table which encapsulated the commits and files visible HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true); HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc); JavaRDD<WriteStatus> statuses = table.compact(jsc, compactionInstant.getTimestamp(), compactionPlan); // Force compaction action statuses.persist(config.getWriteStatusStorageLevel()); // pass extra-metada so that it gets stored in commit file automatically commitCompaction(statuses, table, compactionInstant.getTimestamp(), autoCommit, Optional.ofNullable(compactionPlan.getExtraMetadata())); return statuses; }