@Test public void testMetadataAggregateFromWriteStatus() throws Exception { HoodieWriteConfig cfg = getConfigBuilder(false).withWriteStatusClass(MetadataMergeWriteStatus.class).build(); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); String newCommitTime = "001"; HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(); List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 200); JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1); client.startCommit(); List<WriteStatus> statuses = client.upsert(writeRecords, newCommitTime).collect(); assertNoWriteErrors(statuses); Map<String, String> allWriteStatusMergedMetadataMap = MetadataMergeWriteStatus .mergeMetadataForWriteStatuses(statuses); assertTrue(allWriteStatusMergedMetadataMap.containsKey("InputRecordCount_1506582000")); // For metadata key InputRecordCount_1506582000, value is 2 for each record. So sum of this // should be 2 * records.size() assertEquals(String.valueOf(2 * records.size()), allWriteStatusMergedMetadataMap.get("InputRecordCount_1506582000")); }
assertNoWriteErrors(statuses); assertNoWriteErrors(statuses);
client.commit(newCommitTime, writeStatusJavaRDD); List<WriteStatus> statuses = writeStatusJavaRDD.collect(); assertNoWriteErrors(statuses); assertNoWriteErrors(statuses); statuses = writeStatusJavaRDD.collect(); assertNoWriteErrors(statuses); statuses = writeStatusJavaRDD.collect(); assertNoWriteErrors(statuses);
assertNoWriteErrors(statuses); writeRecords = jsc.parallelize(records, 1); statuses = client.upsert(writeRecords, newCommitTime).collect(); assertNoWriteErrors(statuses); assertNoWriteErrors(statuses);
assertNoWriteErrors(statuses); assertNoWriteErrors(statuses);
assertNoWriteErrors(statuses); assertNoWriteErrors(statuses); metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath()); deltaCommit = metaClient.getActiveTimeline().getDeltaCommitTimeline().lastInstant();