@Test public void testBulkFileWriter() throws IOException { BulkFileProperties bfProperties = (BulkFileProperties) new BulkFileProperties("foo").init(); String filePath = this.getClass().getResource("").getPath() + "/test_bulk_1.csv"; System.out.println("Bulk file path: " + filePath); bfProperties.bulkFilePath.setValue(filePath); bfProperties.schema.schema.setValue(getMakeRowSchema()); // 1.Generate a new file testWriteFile(bfProperties); // 2.Append file bfProperties.append.setValue(true); testWriteFile(bfProperties); // 3.Delete bulk file // deleteBulkFile(bfProperties); }
private void testWriteFile(BulkFileProperties bfProperties) throws IOException { BulkFileSink bulkFileSink = new BulkFileSink(); bulkFileSink.initialize(null, bfProperties); BulkFileWriteOperation writeOperation = (BulkFileWriteOperation) bulkFileSink.createWriteOperation(); Writer<Result> bfWriter = writeOperation.createWriter(null); List<IndexedRecord> rows = makeRows(10); bfWriter.open("foo"); try { for (IndexedRecord row : rows) { bfWriter.write(row); } } catch (Exception e) { e.printStackTrace(); throw e; } finally { Result result = bfWriter.close(); List<Result> results = new ArrayList(); results.add(result); Map<String, Object> resultMap = writeOperation.finalize(results, null); Assert.assertEquals(10, resultMap.get(ComponentDefinition.RETURN_TOTAL_RECORD_COUNT)); } }
private List<IndexedRecord> makeRows(int count) { List<IndexedRecord> outputRows = new ArrayList<>(); for (int i = 0; i < count; i++) { GenericData.Record row = new GenericData.Record(getMakeRowSchema()); row.put("col_1", "aaa_000" + i); row.put("col_2", "bbb_111" + i); row.put("col_3", 123 + i); row.put("col_4", 76543.5 + i); row.put("col_5", new Date()); row.put("col_6", "ddd_333" + i); row.put("col_7", 123456l); System.out.println("Row to write: " // + " col_1: " + row.get("col_1") // + " col_2: " + row.get("col_2") // + " col_3: " + row.get("col_3") // + " col_4: " + row.get("col_4") // + " col_5: " + row.get("col_5") // + " col_6: " + row.get("col_6") + " col_7: " + row.get("col_7")); outputRows.add(row); } return outputRows; }