public static List<HoodieWriteStat> generateFakeHoodieWriteStat(int limit) { List<HoodieWriteStat> writeStatList = new ArrayList<>(); for (int i = 0; i < limit; i++) { HoodieWriteStat writeStat = new HoodieWriteStat(); writeStat.setFileId(UUID.randomUUID().toString()); writeStat.setNumDeletes(0); writeStat.setNumUpdateWrites(100); writeStat.setNumWrites(100); writeStat.setPath("/some/fake/path" + i); writeStat.setPartitionPath("/some/fake/partition/path" + i); writeStat.setTotalLogFilesCompacted(100L); RuntimeStats runtimeStats = new RuntimeStats(); runtimeStats.setTotalScanTime(100); runtimeStats.setTotalCreateTime(100); runtimeStats.setTotalUpsertTime(100); writeStat.setRuntimeStats(runtimeStats); writeStatList.add(writeStat); } return writeStatList; } }
/** * Performs actions to durably, persist the current changes and returns a WriteStatus object */ @Override public WriteStatus close() { logger.info("Closing the file " + status.getFileId() + " as we are done with all the records " + recordsWritten); try { storageWriter.close(); HoodieWriteStat stat = new HoodieWriteStat(); stat.setNumWrites(recordsWritten); stat.setNumDeletes(recordsDeleted); stat.setNumInserts(insertRecordsWritten); stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT); stat.setFileId(status.getFileId()); stat.setPaths(new Path(config.getBasePath()), path, tempPath); stat.setTotalWriteBytes(FSUtils.getFileSize(fs, getStorageWriterPath())); stat.setTotalWriteErrors(status.getFailedRecords().size()); RuntimeStats runtimeStats = new RuntimeStats(); runtimeStats.setTotalCreateTime(timer.endTimer()); stat.setRuntimeStats(runtimeStats); status.setStat(stat); return status; } catch (IOException e) { throw new HoodieInsertException("Failed to close the Insert Handle for path " + path, e); } }
/** * Performs actions to durably, persist the current changes and returns a WriteStatus object */ @Override public WriteStatus close() { logger.info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + recordsWritten); try { storageWriter.close(); HoodieWriteStat stat = new HoodieWriteStat(); stat.setPartitionPath(writeStatus.getPartitionPath()); stat.setNumWrites(recordsWritten); stat.setNumDeletes(recordsDeleted); stat.setNumInserts(insertRecordsWritten); stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT); stat.setFileId(writeStatus.getFileId()); stat.setPaths(new Path(config.getBasePath()), path, tempPath); stat.setTotalWriteBytes(FSUtils.getFileSize(fs, getStorageWriterPath())); stat.setTotalWriteErrors(writeStatus.getFailedRecords().size()); RuntimeStats runtimeStats = new RuntimeStats(); runtimeStats.setTotalCreateTime(timer.endTimer()); stat.setRuntimeStats(runtimeStats); writeStatus.setStat(stat); return writeStatus; } catch (IOException e) { throw new HoodieInsertException("Failed to close the Insert Handle for path " + path, e); } }