@Test public void testRandomData() throws IOException { List<Record> expected = RandomGenericData.generate(SCHEMA, 1000, 435691832918L); File location = temp.newFolder(format.name()); Assert.assertTrue(location.delete()); Table table = TABLES.create(SCHEMA, PartitionSpec.unpartitioned(), ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), location.toString()); AppendFiles append = table.newAppend(); int fileNum = 0; int recordsPerFile = 200; Iterator<Record> iter = expected.iterator(); while (iter.hasNext()) { Path path = new Path(location.toString(), format.addExtension("file-" + fileNum)); int numRecords; List<Record> records = Lists.newArrayList(); for (numRecords = 0; numRecords < recordsPerFile && iter.hasNext(); numRecords += 1) { records.add(iter.next()); } writeFile(location.toString(), format.addExtension("file-" + fileNum), records); append.appendFile(fromInputFile(HadoopInputFile.fromPath(path, CONF), numRecords)); fileNum += 1; } append.commit(); Set<Record> records = Sets.newHashSet(IcebergGenerics.read(table).build()); Assert.assertEquals("Should produce correct number of records", expected.size(), records.size()); Assert.assertEquals("Random record set should match", Sets.newHashSet(expected), records); }
return randomString(random); BigInteger unscaled = randomUnscaled(type.precision(), random); return new BigDecimal(unscaled, type.scale());
protected void writeAndValidate(Schema schema) throws IOException { List<Record> expected = RandomGenericData.generate(schema, 100, 0L); File testFile = temp.newFile(); Assert.assertTrue("Delete should succeed", testFile.delete()); try (FileAppender<Record> appender = Parquet.write(Files.localOutput(testFile)) .schema(schema) .createWriterFunc(GenericParquetWriter::buildWriter) .build()) { appender.addAll(expected); } List<Record> rows; try (CloseableIterable<Record> reader = Parquet.read(Files.localInput(testFile)) .project(schema) .createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema)) .build()) { rows = Lists.newArrayList(reader); } for (int i = 0; i < expected.size(); i += 1) { DataTestHelpers.assertEquals(schema.asStruct(), expected.get(i), rows.get(i)); } } }
protected void writeAndValidate(Schema schema) throws IOException { List<Record> expected = RandomGenericData.generate(schema, 100, 0L); File testFile = temp.newFile(); Assert.assertTrue("Delete should succeed", testFile.delete()); try (FileAppender<Record> writer = Avro.write(Files.localOutput(testFile)) .schema(schema) .createWriterFunc(DataWriter::create) .named("test") .build()) { for (Record rec : expected) { writer.add(rec); } } List<Record> rows; try (AvroIterable<Record> reader = Avro.read(Files.localInput(testFile)) .project(schema) .createReaderFunc(DataReader::create) .build()) { rows = Lists.newArrayList(reader); } for (int i = 0; i < expected.size(); i += 1) { DataTestHelpers.assertEquals(schema.asStruct(), expected.get(i), rows.get(i)); } } }