private HoodieWriteConfig.Builder getConfigBuilder(Boolean autoCommit) { return getConfigBuilder(autoCommit, IndexType.BLOOM); }
@Test public void testUpsertPartitioner() throws Exception { HoodieWriteConfig cfg = getConfig(true); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); assertNoWriteErrors(statuses); assertNoWriteErrors(statuses);
@Test public void testMetadataAggregateFromWriteStatus() throws Exception { HoodieWriteConfig cfg = getConfigBuilder(false).withWriteStatusClass(MetadataMergeWriteStatus.class).build(); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); String newCommitTime = "001"; HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(); List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 200); JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1); client.startCommit(); List<WriteStatus> statuses = client.upsert(writeRecords, newCommitTime).collect(); assertNoWriteErrors(statuses); Map<String, String> allWriteStatusMergedMetadataMap = MetadataMergeWriteStatus .mergeMetadataForWriteStatuses(statuses); assertTrue(allWriteStatusMergedMetadataMap.containsKey("InputRecordCount_1506582000")); // For metadata key InputRecordCount_1506582000, value is 2 for each record. So sum of this // should be 2 * records.size() assertEquals(String.valueOf(2 * records.size()), allWriteStatusMergedMetadataMap.get("InputRecordCount_1506582000")); }
@Test public void testLogFileCountsAfterCompaction() throws Exception { HoodieWriteConfig config = getConfig(true); HoodieWriteClient writeClient = new HoodieWriteClient(jsc, config); HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
@Test public void testRollbackWithDeltaAndCompactionCommit() throws Exception { HoodieWriteConfig cfg = getConfig(false); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); client.commit(newCommitTime, writeStatusJavaRDD); List<WriteStatus> statuses = writeStatusJavaRDD.collect(); assertNoWriteErrors(statuses); assertNoWriteErrors(statuses); statuses = writeStatusJavaRDD.collect(); assertNoWriteErrors(statuses); statuses = writeStatusJavaRDD.collect(); assertNoWriteErrors(statuses);
@Test public void testMetadataValuesAfterInsertUpsertAndCompaction() throws Exception { HoodieWriteConfig config = getConfig(false); HoodieWriteClient writeClient = new HoodieWriteClient(jsc, config); HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
@Test public void testSimpleInsertUpdateAndDelete() throws Exception { HoodieWriteConfig cfg = getConfig(true); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); assertNoWriteErrors(statuses); writeRecords = jsc.parallelize(records, 1); statuses = client.upsert(writeRecords, newCommitTime).collect(); assertNoWriteErrors(statuses); assertNoWriteErrors(statuses);
private HoodieWriteConfig getConfig(Boolean autoCommit) { return getConfigBuilder(autoCommit).build(); }
HoodieWriteConfig cfg = getConfig(true); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); assertNoWriteErrors(statuses); assertNoWriteErrors(statuses);
HoodieWriteConfig config = getConfigBuilder(false, IndexType.INMEMORY).build(); HoodieWriteClient writeClient = new HoodieWriteClient(jsc, config); HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
@Test public void testSimpleInsertAndUpdate() throws Exception { HoodieWriteConfig cfg = getConfig(true); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); assertNoWriteErrors(statuses); assertNoWriteErrors(statuses); metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath()); deltaCommit = metaClient.getActiveTimeline().getDeltaCommitTimeline().lastInstant();
HoodieWriteConfig config = getConfigBuilder(false, IndexType.INMEMORY).build(); HoodieWriteClient writeClient = new HoodieWriteClient(jsc, config); HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
HoodieWriteConfig config = getConfigBuilder(false, IndexType.INMEMORY).build(); HoodieWriteClient writeClient = new HoodieWriteClient(jsc, config); HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
public void testRollingStatsWithSmallFileHandling() throws Exception { HoodieWriteConfig cfg = getConfigBuilder(false, IndexType.INMEMORY).withAutoCommit(false).build(); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath);
public void testRollingStatsInMetadata() throws Exception { HoodieWriteConfig cfg = getConfigBuilder(false, IndexType.INMEMORY).withAutoCommit(false).build(); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath);