@Test @Category(NeedsRunner.class) public void testFlatSchema() { List<Row> rows = IntStream.rangeClosed(0, 2) .mapToObj(i -> Row.withSchema(SIMPLE_SCHEMA).addValues(i, Integer.toString(i)).build()) .collect(Collectors.toList()); PCollection<Row> unnested = pipeline.apply(Create.of(rows).withRowSchema(SIMPLE_SCHEMA)).apply(Unnest.create()); PAssert.that(unnested).containsInAnyOrder(rows); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testClashingNamePolicy() { List<Row> bottomRow = IntStream.rangeClosed(0, 2) .mapToObj(i -> Row.withSchema(SIMPLE_SCHEMA).addValues(i, Integer.toString(i)).build()) .collect(Collectors.toList()); thrown.expect(IllegalArgumentException.class); List<Row> rows = bottomRow .stream() .map(r -> Row.withSchema(NESTED_SCHEMA).addValues(r, r).build()) .collect(Collectors.toList()); PCollection<Row> unnested = pipeline .apply(Create.of(rows).withRowSchema(NESTED_SCHEMA)) .apply(Unnest.<Row>create().withFieldNameFunction(Unnest.KEEP_NESTED_NAME)); pipeline.run(); } }
@Test @Category(NeedsRunner.class) public void testAlternateNamePolicy() { List<Row> bottomRow = IntStream.rangeClosed(0, 2) .mapToObj(i -> Row.withSchema(SIMPLE_SCHEMA).addValues(i, Integer.toString(i)).build()) .collect(Collectors.toList()); List<Row> rows = bottomRow .stream() .map(r -> Row.withSchema(NESTED_SCHEMA2).addValues(r).build()) .collect(Collectors.toList()); PCollection<Row> unnested = pipeline .apply(Create.of(rows).withRowSchema(NESTED_SCHEMA2)) .apply(Unnest.<Row>create().withFieldNameFunction(Unnest.KEEP_NESTED_NAME)); assertEquals(UNNESTED2_SCHEMA_ALTERNATE, unnested.getSchema()); List<Row> expected = bottomRow .stream() .map( r -> Row.withSchema(UNNESTED2_SCHEMA_ALTERNATE) .addValues(r.getValue(0), r.getValue(1)) .build()) .collect(Collectors.toList()); ; PAssert.that(unnested).containsInAnyOrder(expected); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testSimpleUnnesting() { List<Row> bottomRow = IntStream.rangeClosed(0, 2) .mapToObj(i -> Row.withSchema(SIMPLE_SCHEMA).addValues(i, Integer.toString(i)).build()) .collect(Collectors.toList()); List<Row> rows = bottomRow .stream() .map(r -> Row.withSchema(NESTED_SCHEMA).addValues(r, r).build()) .collect(Collectors.toList()); PCollection<Row> unnested = pipeline.apply(Create.of(rows).withRowSchema(NESTED_SCHEMA)).apply(Unnest.create()); assertEquals(UNNESTED_SCHEMA, unnested.getSchema()); List<Row> expected = bottomRow .stream() .map( r -> Row.withSchema(UNNESTED_SCHEMA) .addValues(r.getValue(0), r.getValue(1), r.getValue(0), r.getValue(1)) .build()) .collect(Collectors.toList()); ; PAssert.that(unnested).containsInAnyOrder(expected); pipeline.run(); }