/** * Build Compaction operation payload from Avro version for using in Spark executors * * @param hc HoodieCompactionOperation */ public static CompactionOperation buildCompactionOperation(HoodieCompactionOperation hc) { return CompactionOperation.convertFromAvroRecordInstance(hc); }
/** * Create a file system view, as of the given timeline */ public HoodieTableFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline) { this.metaClient = metaClient; this.visibleActiveTimeline = visibleActiveTimeline; this.fileGroupMap = new HashMap<>(); this.partitionToFileGroupsMap = new HashMap<>(); // Build fileId to Pending Compaction Instants List<HoodieInstant> pendingCompactionInstants = metaClient.getActiveTimeline().filterPendingCompactionTimeline().getInstants().collect(Collectors.toList()); this.fileIdToPendingCompaction = ImmutableMap.copyOf( CompactionUtils.getAllPendingCompactionOperations(metaClient).entrySet().stream() .map(entry -> Pair.of(entry.getKey(), Pair.of(entry.getValue().getKey(), CompactionOperation.convertFromAvroRecordInstance(entry.getValue().getValue())))) .collect(Collectors.toMap(Pair::getKey, Pair::getValue))); }
/** * Generate renaming actions for unscheduling a fileId from pending compaction. NOTE: Can only be used safely when no * writer (ingestion/compaction) is running. * * @param metaClient Hoodie Table MetaClient * @param fileId FileId to remove compaction * @param fsViewOpt Cached File System View * @param skipValidation Skip Validation * @return list of pairs of log-files (old, new) and for each pair, rename must be done to successfully unschedule * compaction. */ public List<Pair<HoodieLogFile, HoodieLogFile>> getRenamingActionsForUnschedulingCompactionForFileId( HoodieTableMetaClient metaClient, String fileId, Optional<HoodieTableFileSystemView> fsViewOpt, boolean skipValidation) throws IOException { Map<String, Pair<String, HoodieCompactionOperation>> allPendingCompactions = CompactionUtils.getAllPendingCompactionOperations(metaClient); if (allPendingCompactions.containsKey(fileId)) { Pair<String, HoodieCompactionOperation> opWithInstant = allPendingCompactions.get(fileId); return getRenamingActionsForUnschedulingCompactionOperation(metaClient, opWithInstant.getKey(), CompactionOperation.convertFromAvroRecordInstance(opWithInstant.getValue()), fsViewOpt, skipValidation); } throw new HoodieException("FileId " + fileId + " not in pending compaction"); }
@Test public void testUnscheduleCompactionFileId() throws Exception { int numEntriesPerInstant = 10; CompactionTestUtils .setupAndValidateCompactionOperations(metaClient, false, numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant); Map<String, CompactionOperation> instantsWithOp = Arrays.asList("001", "003", "005", "007").stream().map(instant -> { try { return Pair.of(instant, CompactionUtils.getCompactionPlan(metaClient, instant)); } catch (IOException ioe) { throw new HoodieException(ioe); } }).map(instantWithPlan -> instantWithPlan.getRight().getOperations().stream().map(op -> Pair.of( instantWithPlan.getLeft(), CompactionOperation.convertFromAvroRecordInstance(op))).findFirst().get()) .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); // THere are delta-commits after compaction instant validateUnScheduleFileId(client, "000", "001", instantsWithOp.get("001"), 2); // THere are delta-commits after compaction instant validateUnScheduleFileId(client, "002", "003", instantsWithOp.get("003"), 2); // THere are no delta-commits after compaction instant validateUnScheduleFileId(client, "004", "005", instantsWithOp.get("005"), 0); // THere are no delta-commits after compaction instant validateUnScheduleFileId(client, "006", "007", instantsWithOp.get("007"), 0); }