private void runSingleFilter(PCollection<Row> input) throws Exception { String sql = "SELECT * FROM PCOLLECTION WHERE f_int = 1"; PCollection<Row> result = input.apply("testSingleFilter", SqlTransform.query(sql)); PAssert.that(result).containsInAnyOrder(rowsInTableA.get(0)); pipeline.run().waitUntilFinish(); }
@Test public void countAssertsMultipleCallsIndependent() { PCollection<Integer> create = pipeline.apply("FirstCreate", Create.of(1, 2, 3)); PAssert.that(create).containsInAnyOrder(1, 2, 3); PAssert.thatSingleton(create.apply(Sum.integersGlobally())).isEqualTo(6); assertThat(PAssert.countAsserts(pipeline), equalTo(2)); PAssert.thatMap(pipeline.apply("CreateMap", Create.of(KV.of(1, 2)))) .isEqualTo(Collections.singletonMap(1, 2)); assertThat(PAssert.countAsserts(pipeline), equalTo(3)); } }
@Test @Category(NeedsRunner.class) @SuppressWarnings("unchecked") public void testTop() { PCollection<String> input = p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of())); PCollection<List<String>> top1 = input.apply(Top.of(1, new OrderByLength())); PCollection<List<String>> top2 = input.apply(Top.largest(2)); PCollection<List<String>> top3 = input.apply(Top.smallest(3)); PCollection<KV<String, Integer>> inputTable = createInputTable(p); PCollection<KV<String, List<Integer>>> largestPerKey = inputTable.apply(Top.largestPerKey(2)); PCollection<KV<String, List<Integer>>> smallestPerKey = inputTable.apply(Top.smallestPerKey(2)); PAssert.thatSingletonIterable(top1).containsInAnyOrder(Arrays.asList("bb")); PAssert.thatSingletonIterable(top2).containsInAnyOrder("z", "c"); PAssert.thatSingletonIterable(top3).containsInAnyOrder("a", "bb", "c"); PAssert.that(largestPerKey) .containsInAnyOrder(KV.of("a", Arrays.asList(3, 2)), KV.of("b", Arrays.asList(100, 10))); PAssert.that(smallestPerKey) .containsInAnyOrder(KV.of("a", Arrays.asList(1, 2)), KV.of("b", Arrays.asList(1, 10))); p.run(); }
.withCoder(SerializableCoder.of(TestRow.class))); PAssert.thatSingleton(namesAndIds.apply("Count All", Count.globally())) .isEqualTo((long) numberOfRows); .apply(ParDo.of(new TestRow.SelectNameFn())) .apply("Hash row contents", Combine.globally(new HashingFn()).withoutDefaults()); PAssert.that(consolidatedHashcode) .containsInAnyOrder(TestRow.getExpectedHashForRowCount(numberOfRows)); PAssert.thatSingletonIterable(frontOfList).containsInAnyOrder(expectedFrontOfList); PAssert.thatSingletonIterable(backOfList).containsInAnyOrder(expectedBackOfList);
/** * Constructs a {@link SingletonAssert} for the value of the provided {@code PCollection * PCollection<T>}, which must be a singleton. */ public static <T> SingletonAssert<T> thatSingleton(PCollection<T> actual) { return thatSingleton(actual.getName(), actual); }
/** * Constructs a {@link SingletonAssert} for the value of the provided {@link PCollection}, which * must have at most one value per key. * * <p>Note that the actual value must be coded by a {@link KvCoder}, not just any {@code Coder<K, * V>}. */ public static <K, V> SingletonAssert<Map<K, V>> thatMap(PCollection<KV<K, V>> actual) { return thatMap(actual.getName(), actual); }
/** * Constructs an {@link IterableAssert} for the value of the provided {@link PCollection} which * must contain a single {@code Iterable<T>} value. */ public static <T> IterableAssert<T> thatSingletonIterable( PCollection<? extends Iterable<T>> actual) { return thatSingletonIterable(actual.getName(), actual); }
@VisibleForTesting void updatePAssertCount(Pipeline pipeline) { if (hasExperiment(options, "beam_fn_api")) { // TODO[BEAM-1866]: FnAPI does not support metrics, so expect 0 assertions. expectedNumberOfAssertions = 0; } else { expectedNumberOfAssertions = PAssert.countAsserts(pipeline); } }
public static void addCountingAsserts(PCollection<Long> input, long start, long end) { // Count == numElements PAssert.thatSingleton(input.apply("Count", Count.globally())).isEqualTo(end - start); // Unique count == numElements PAssert.thatSingleton(input.apply(Distinct.create()).apply("UniqueCount", Count.globally())) .isEqualTo(end - start); // Min == start PAssert.thatSingleton(input.apply("Min", Min.globally())).isEqualTo(start); // Max == end-1 PAssert.thatSingleton(input.apply("Max", Max.globally())).isEqualTo(end - 1); }
@Test @Category(ValidatesRunner.class) public void testCoGroupByKeyGetOnly() { final TupleTag<String> tag1 = new TupleTag<>(); final TupleTag<String> tag2 = new TupleTag<>(); PCollection<KV<Integer, CoGbkResult>> coGbkResults = buildGetOnlyGbk(p, tag1, tag2); PAssert.thatMap(coGbkResults) .satisfies( results -> { assertEquals("collection1-1", results.get(1).getOnly(tag1)); assertEquals("collection1-2", results.get(2).getOnly(tag1)); assertEquals("collection2-2", results.get(2).getOnly(tag2)); assertEquals("collection2-3", results.get(3).getOnly(tag2)); return null; }); p.run(); }
@Test @Category(NeedsRunner.class) public void testSampleEmpty() { PCollection<Integer> input = pipeline.apply(Create.empty(BigEndianIntegerCoder.of())); PCollection<Iterable<Integer>> output = input.apply(Sample.fixedSizeGlobally(3)); PAssert.thatSingletonIterable(output).satisfies(new VerifyCorrectSample<>(0, EMPTY)); pipeline.run(); }
long expectedNumberOfAssertions = (long) PAssert.countAsserts(pipeline);
private void runCompositeFilter(PCollection<Row> input) throws Exception { String sql = "SELECT * FROM TABLE_A" + " WHERE f_int > 1 AND (f_long < 3000 OR f_string = 'string_row3')"; PCollection<Row> result = PCollectionTuple.of(new TupleTag<>("TABLE_A"), input) .apply("testCompositeFilter", SqlTransform.query(sql)); PAssert.that(result).containsInAnyOrder(rowsInTableA.get(1), rowsInTableA.get(2)); pipeline.run().waitUntilFinish(); }
@Test public void countAssertsSucceeds() { PCollection<Integer> create = pipeline.apply("FirstCreate", Create.of(1, 2, 3)); PAssert.that(create).containsInAnyOrder(1, 2, 3); PAssert.thatSingleton(create.apply(Sum.integersGlobally())).isEqualTo(6); PAssert.thatMap(pipeline.apply("CreateMap", Create.of(KV.of(1, 2)))) .isEqualTo(Collections.singletonMap(1, 2)); assertThat(PAssert.countAsserts(pipeline), equalTo(3)); }
public static void addCountingAsserts(PCollection<Long> input, long numElements) { // Count == numElements PAssert.thatSingleton(input.apply("Count", Count.globally())).isEqualTo(numElements); // Unique count == numElements PAssert.thatSingleton(input.apply(Distinct.create()).apply("UniqueCount", Count.globally())) .isEqualTo(numElements); // Min == 0 PAssert.thatSingleton(input.apply("Min", Min.globally())).isEqualTo(0L); // Max == numElements-1 PAssert.thatSingleton(input.apply("Max", Max.globally())).isEqualTo(numElements - 1); }
@Test @Category(NeedsRunner.class) @SuppressWarnings("unchecked") public void testTopEmpty() { PCollection<String> input = p.apply(Create.of(Arrays.asList(EMPTY_COLLECTION)).withCoder(StringUtf8Coder.of())); PCollection<List<String>> top1 = input.apply(Top.of(1, new OrderByLength())); PCollection<List<String>> top2 = input.apply(Top.largest(2)); PCollection<List<String>> top3 = input.apply(Top.smallest(3)); PCollection<KV<String, Integer>> inputTable = createEmptyInputTable(p); PCollection<KV<String, List<Integer>>> largestPerKey = inputTable.apply(Top.largestPerKey(2)); PCollection<KV<String, List<Integer>>> smallestPerKey = inputTable.apply(Top.smallestPerKey(2)); PAssert.thatSingletonIterable(top1).empty(); PAssert.thatSingletonIterable(top2).empty(); PAssert.thatSingletonIterable(top3).empty(); PAssert.that(largestPerKey).empty(); PAssert.that(smallestPerKey).empty(); p.run(); }
buildPurchasesCoGbk(p, purchasesTag, addressesTag, namesTag); PAssert.thatMap(coGbkResults) .satisfies( results -> {
@Test @Category(NeedsRunner.class) public void testSample() { PCollection<Integer> input = pipeline.apply( Create.of(ImmutableList.copyOf(DATA)).withCoder(BigEndianIntegerCoder.of())); PCollection<Iterable<Integer>> output = input.apply(Sample.fixedSizeGlobally(3)); PAssert.thatSingletonIterable(output).satisfies(new VerifyCorrectSample<>(3, DATA)); pipeline.run(); }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testEmptyFalse() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); PAssert.that("Vals should have been empty", vals).empty(); Throwable thrown = runExpectingAssertionFailure(pipeline); String message = thrown.getMessage(); assertThat(message, containsString("Vals should have been empty")); assertThat(message, containsString("Expected: iterable over [] in any order")); }