/** * If it is able to update {@link #saveChanges} from true to false; then only it will create new hoodie * commit and will save metadata information in it. */ public void saveChanges() { if (!this.saveChanges.compareAndSet(true, false)) { log.info("Metadata info is already saved. Not saving it again."); return; } final HoodieWriteClient<HoodieAvroPayload> writeClient = new HoodieWriteClient<>(jsc.get(), this.hoodieConf.getHoodieWriteConfig(), true); final String commitTime = writeClient.startCommit(); log.info("Saving metadata info using hoodie-commit: {}", commitTime); final List<WriteStatus> dummyWrites = new ArrayList<>(); final boolean ret = writeClient .commit(commitTime, jsc.get().parallelize(dummyWrites), java.util.Optional.of(getMetadataInfo())); if (!ret) { throw new JobRuntimeException("Failed to save metadata information."); } }
public void commit(@NonNull final HoodieWriteClientWrapper hoodieWriteClient, @NotEmpty final String commitTime, @NonNull final Optional<JavaRDD<WriteStatus>> writesStatuses, final boolean shouldSaveChangesInFuture) { updateSinkStat(writesStatuses); logWriteMetrics(writesStatuses); java.util.Optional<HashMap<String, String>> hoodieExtraMetadata = java.util.Optional.empty(); if (this.metadataMgr instanceof HoodieBasedMetadataManager) { // Retrieve metadata from metadata manager and update metadata manager to avoid it creating extra // hoodie commit. final HoodieBasedMetadataManager hoodieBasedMetadataManager = (HoodieBasedMetadataManager) this.metadataMgr; hoodieExtraMetadata = java.util.Optional.of(hoodieBasedMetadataManager.getMetadataInfo()); if (!shouldSaveChangesInFuture) { hoodieBasedMetadataManager.shouldSaveChanges().set(false); } } hoodieSinkOperations.preCommitOperations(this.hoodieConf, commitTime); if (writesStatuses.isPresent() || hoodieExtraMetadata.isPresent()) { if (writesStatuses.isPresent()) { hoodieWriteClient.commit(commitTime, writesStatuses.get(), hoodieExtraMetadata); } else { hoodieWriteClient.commit(commitTime, this.jsc.emptyRDD(), hoodieExtraMetadata); } } }
Mockito.verify(hoodieWriteClientWrapper, Mockito.times(1)) .commit(Matchers.anyString(), Matchers.any(JavaRDD.class), Matchers.eq(java.util.Optional.of(hoodieBasedMetadataManager.getMetadataInfo()))); Mockito.verify(hoodieWriteClientWrapper, Mockito.times(1)).close(); Assert.assertFalse(hoodieBasedMetadataManager.shouldSaveChanges().get());
Mockito.verify(hoodieWriteClientWrapper, Mockito.times(1)) .commit(Matchers.anyString(), Matchers.any(JavaRDD.class), Matchers.eq(java.util.Optional.of(hoodieBasedMetadataManager.getMetadataInfo()))); Assert.assertFalse(hoodieBasedMetadataManager.shouldSaveChanges().get());
Assert.assertTrue(mgr.getMetadataInfo().get(HoodieBasedMetadataManager.HOODIE_METADATA_KEY).isEmpty()); Assert.assertFalse(mgr.getMetadataInfo().get(HoodieBasedMetadataManager.HOODIE_METADATA_KEY).isEmpty()); new HoodieBasedMetadataManager(hoodieConf, condition2, this.jsc.get()); Assert.assertEquals(1, mgr2.getAll().size()); Assert.assertFalse(mgr2.getMetadataInfo().get(HoodieBasedMetadataManager.HOODIE_METADATA_KEY).isEmpty()); Assert.assertEquals(mgr.getMetadataInfo(), mgr2.getMetadataInfo());