@Test public void testRandomData() throws IOException { List<Record> expected = RandomGenericData.generate(SCHEMA, 1000, 435691832918L); File location = temp.newFolder(format.name()); Assert.assertTrue(location.delete()); Table table = TABLES.create(SCHEMA, PartitionSpec.unpartitioned(), ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), location.toString()); AppendFiles append = table.newAppend(); int fileNum = 0; int recordsPerFile = 200; Iterator<Record> iter = expected.iterator(); while (iter.hasNext()) { Path path = new Path(location.toString(), format.addExtension("file-" + fileNum)); int numRecords; List<Record> records = Lists.newArrayList(); for (numRecords = 0; numRecords < recordsPerFile && iter.hasNext(); numRecords += 1) { records.add(iter.next()); } writeFile(location.toString(), format.addExtension("file-" + fileNum), records); append.appendFile(fromInputFile(HadoopInputFile.fromPath(path, CONF), numRecords)); fileNum += 1; } append.commit(); Set<Record> records = Sets.newHashSet(IcebergGenerics.read(table).build()); Assert.assertEquals("Should produce correct number of records", expected.size(), records.size()); Assert.assertEquals("Random record set should match", Sets.newHashSet(expected), records); }
record.copy(ImmutableMap.of("id", 2L, "data", "falafel")) ); InputFile file1 = writeFile(sharedTableLocation, format.addExtension("file-1"), file1Records); record.copy(ImmutableMap.of("id", 12L, "data", "tissue")) ); InputFile file2 = writeFile(sharedTableLocation, format.addExtension("file-2"), file2Records); record.copy(ImmutableMap.of("id", 22L, "data", "preventative")) ); InputFile file3 = writeFile(sharedTableLocation, format.addExtension("file-3"), file3Records); ImmutableMap.of(1, longToBuffer(0L)), // lower bounds ImmutableMap.of(1, longToBuffer(2L)))) // upper bounds) .build()) .appendFile(DataFiles.builder(PartitionSpec.unpartitioned()) ImmutableMap.of(1, longToBuffer(10L)), // lower bounds ImmutableMap.of(1, longToBuffer(12L)))) // upper bounds) .build()) .appendFile(DataFiles.builder(PartitionSpec.unpartitioned()) ImmutableMap.of(1, longToBuffer(20L)), // lower bounds ImmutableMap.of(1, longToBuffer(22L)))) // upper bounds) .build()) .commit();