public long fetchTotalFilesUpdated() { long totalFilesUpdated = 0; for (List<HoodieWriteStat> stats : partitionToWriteStats.values()) { for (HoodieWriteStat stat : stats) { if (stat.getPrevCommit() != null && !stat.getPrevCommit().equals("null")) { totalFilesUpdated++; } } } return totalFilesUpdated; }
public long fetchTotalFilesInsert() { long totalFilesInsert = 0; for (List<HoodieWriteStat> stats : partitionToWriteStats.values()) { for (HoodieWriteStat stat : stats) { if (stat.getPrevCommit() != null && stat.getPrevCommit().equals("null")) { totalFilesInsert++; } } } return totalFilesInsert; }
public long fetchTotalInsertRecordsWritten() { long totalInsertRecordsWritten = 0; for (List<HoodieWriteStat> stats : partitionToWriteStats.values()) { for (HoodieWriteStat stat : stats) { if (stat.getPrevCommit() != null && stat.getPrevCommit().equals("null")) { totalInsertRecordsWritten += stat.getNumWrites(); } } } return totalInsertRecordsWritten; }
.filter(wStat -> { return wStat != null && wStat.getPrevCommit() != HoodieWriteStat.NULL_COMMIT && wStat.getPrevCommit() != null && !deletedFiles.contains(wStat.getFileId()); }).forEach(wStat -> { HoodieLogFormat.Writer writer = null; String baseCommitTime = wStat.getPrevCommit(); if (hoodieIndex.isGlobal()) { baseCommitTime = fileIdToBaseCommitTimeForLogMap.get(wStat.getFileId());
.filter(wStat -> { if (wStat != null && wStat.getPrevCommit() != HoodieWriteStat.NULL_COMMIT && wStat.getPrevCommit() != null && !deletedFiles.contains(wStat.getFileId())) { return true; }).forEach(wStat -> { HoodieLogFormat.Writer writer = null; String baseCommitTime = wStat.getPrevCommit(); if (hoodieIndex.isGlobal()) { baseCommitTime = fileIdToBaseCommitTimeForLogMap.get(wStat.getFileId());
long totalWriteErrors = 0; for (HoodieWriteStat stat : stats) { if (stat.getPrevCommit().equals(HoodieWriteStat.NULL_COMMIT)) { totalFilesAdded += 1; totalRecordsInserted += stat.getNumWrites();
List<HoodieWriteStat> stats = entry.getValue(); for (HoodieWriteStat stat : stats) { rows.add(new Comparable[]{path, stat.getFileId(), stat.getPrevCommit(), stat.getNumUpdateWrites(), stat.getNumWrites(), stat.getTotalWriteBytes(), stat.getTotalWriteErrors()});
.filter(status -> status.getStat().getPrevCommit() != HoodieWriteStat.NULL_COMMIT).count() > 0); .filter(status -> status.getStat().getPrevCommit() == HoodieWriteStat.NULL_COMMIT).count(), 0); .filter(status -> status.getStat().getPrevCommit() == HoodieWriteStat.NULL_COMMIT).count(), 0);
assertEquals("Existing file should be expanded", commitTime1, statuses.get(0).getStat().getPrevCommit()); Path newFile = new Path(basePath, testPartitionPath + "/" + FSUtils.makeDataFileName(commitTime2, 0, file1)); assertEquals("file should contain 140 records",
assertEquals("Existing file should be expanded", commitTime1, statuses.get(0).getStat().getPrevCommit()); Path newFile = new Path(basePath, testPartitionPath + "/" + FSUtils.makeDataFileName(commitTime2, 0, file1)); assertEquals("file should contain 140 records",