public void commit(@NonNull final HoodieWriteClientWrapper hoodieWriteClient, @NotEmpty final String commitTime, @NonNull final Optional<JavaRDD<WriteStatus>> writesStatuses, final boolean shouldSaveChangesInFuture) { updateSinkStat(writesStatuses); logWriteMetrics(writesStatuses); java.util.Optional<HashMap<String, String>> hoodieExtraMetadata = java.util.Optional.empty(); if (this.metadataMgr instanceof HoodieBasedMetadataManager) { // Retrieve metadata from metadata manager and update metadata manager to avoid it creating extra // hoodie commit. final HoodieBasedMetadataManager hoodieBasedMetadataManager = (HoodieBasedMetadataManager) this.metadataMgr; hoodieExtraMetadata = java.util.Optional.of(hoodieBasedMetadataManager.getMetadataInfo()); if (!shouldSaveChangesInFuture) { hoodieBasedMetadataManager.shouldSaveChanges().set(false); } } hoodieSinkOperations.preCommitOperations(this.hoodieConf, commitTime); if (writesStatuses.isPresent() || hoodieExtraMetadata.isPresent()) { if (writesStatuses.isPresent()) { hoodieWriteClient.commit(commitTime, writesStatuses.get(), hoodieExtraMetadata); } else { hoodieWriteClient.commit(commitTime, this.jsc.emptyRDD(), hoodieExtraMetadata); } } }
/** * @param shouldSaveChanges {@link AtomicBoolean} which {@link #saveChanges} will use to determine if * it should create new commit and save changes or not. It will save changes into new commit only if CAS operation * succeeds in setting it to true (from false). * @param hoodieConf {@link HoodieConfiguration} */ public HoodieBasedMetadataManager(@NonNull final HoodieConfiguration hoodieConf, @NonNull final AtomicBoolean shouldSaveChanges, @NonNull final JavaSparkContext jsc) throws IOException { this.hoodieConf = hoodieConf; this.saveChanges = shouldSaveChanges; this.jsc = Optional.of(jsc); this.metadataMap = readMetadataInfo(this.hoodieConf); }
HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final HoodieBasedMetadataManager hoodieBasedMetadataManager = new HoodieBasedMetadataManager(hoodieConf, new AtomicBoolean(true), this.jsc.get()); hoodieBasedMetadataManager.set("randomKey", new StringValue("randomValue")); final MockHoodieSink hoodieSink = new MockHoodieSink(hoodieConf, hoodieKeyGenerator, jsc.get(), UPSERT, hoodieBasedMetadataManager); Mockito.verify(hoodieWriteClientWrapper, Mockito.times(1)) .commit(Matchers.anyString(), Matchers.any(JavaRDD.class), Matchers.eq(java.util.Optional.of(hoodieBasedMetadataManager.getMetadataInfo()))); Assert.assertFalse(hoodieBasedMetadataManager.shouldSaveChanges().get());
final AtomicBoolean condition = new AtomicBoolean(true); final HoodieBasedMetadataManager mgr = new HoodieBasedMetadataManager(hoodieConf, condition, this.jsc.get()); Assert.assertEquals(0, mgr.getAll().size()); Assert.assertTrue(mgr.getMetadataInfo().get(HoodieBasedMetadataManager.HOODIE_METADATA_KEY).isEmpty()); mgr.set(testKey, new StringValue(testValue)); Assert.assertEquals(1, mgr.getAll().size()); Assert.assertFalse(mgr.getMetadataInfo().get(HoodieBasedMetadataManager.HOODIE_METADATA_KEY).isEmpty()); mgr.saveChanges(); mgr.saveChanges(); Assert.assertFalse(condition.get()); Assert.assertFalse( new HoodieBasedMetadataManager(hoodieConf, condition2, this.jsc.get()); Assert.assertEquals(1, mgr2.getAll().size()); Assert.assertFalse(mgr2.getMetadataInfo().get(HoodieBasedMetadataManager.HOODIE_METADATA_KEY).isEmpty()); Assert.assertEquals(mgr.getMetadataInfo(), mgr2.getMetadataInfo());
.enableMetrics(false).build(); final AtomicBoolean condition = new AtomicBoolean(true); final HoodieBasedMetadataManager mgr = new HoodieBasedMetadataManager(hoodieConf, condition, this.jsc.get()); mgr.set(testKey, new StringValue(testValue)); mgr.saveChanges(); final HoodieBasedMetadataManager mgr2 = new HoodieBasedMetadataManager(hoodieConf, condition, this.jsc.get()); Assert.assertEquals(testValue, mgr2.get(testKey).get().getValue()); mgr2.remove(testKey); Assert.assertFalse(mgr2.get(testKey).isPresent()); final HoodieBasedMetadataManager mgr3 = new HoodieBasedMetadataManager(hoodieConf, condition, this.jsc.get()); Assert.assertEquals(testValue, mgr3.get(testKey).get().getValue()); mgr2.saveChanges(); final HoodieBasedMetadataManager mgr4 = new HoodieBasedMetadataManager(hoodieConf, condition, this.jsc.get()); Assert.assertFalse(mgr4.get(testKey).isPresent());
new HoodieBasedMetadataManager(hoodieConf, shouldSaveChanges, jsc); final HoodieSink hoodieSink = new HoodieErrorSink(hoodieConf, new DummyHoodieSinkDataConverter(), jsc, HoodieSink.HoodieSinkOp.BULK_INSERT, metadataManager,
/** * If it is able to update {@link #saveChanges} from true to false; then only it will create new hoodie * commit and will save metadata information in it. */ public void saveChanges() { if (!this.saveChanges.compareAndSet(true, false)) { log.info("Metadata info is already saved. Not saving it again."); return; } final HoodieWriteClient<HoodieAvroPayload> writeClient = new HoodieWriteClient<>(jsc.get(), this.hoodieConf.getHoodieWriteConfig(), true); final String commitTime = writeClient.startCommit(); log.info("Saving metadata info using hoodie-commit: {}", commitTime); final List<WriteStatus> dummyWrites = new ArrayList<>(); final boolean ret = writeClient .commit(commitTime, jsc.get().parallelize(dummyWrites), java.util.Optional.of(getMetadataInfo())); if (!ret) { throw new JobRuntimeException("Failed to save metadata information."); } }
HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final HoodieBasedMetadataManager hoodieBasedMetadataManager = new HoodieBasedMetadataManager(hoodieConf, new AtomicBoolean(true), this.jsc.get()); hoodieBasedMetadataManager.set("randomKey", new StringValue("randomValue")); final MockHoodieSink hoodieSink = new MockHoodieSink(hoodieConf, hoodieKeyGenerator, jsc.get(), INSERT, hoodieBasedMetadataManager); Mockito.verify(hoodieWriteClientWrapper, Mockito.times(1)) .commit(Matchers.anyString(), Matchers.any(JavaRDD.class), Matchers.eq(java.util.Optional.of(hoodieBasedMetadataManager.getMetadataInfo()))); Mockito.verify(hoodieWriteClientWrapper, Mockito.times(1)).close(); Assert.assertFalse(hoodieBasedMetadataManager.shouldSaveChanges().get());
.build(); final IMetadataManager metadataMgr = new HoodieBasedMetadataManager(hoodieConf, new AtomicBoolean(true), jsc); final HoodieSink hoodieSink = new HoodieSink( hoodieConf,