@Test public void testViewUnboundedAsMultimapDirect() { testViewUnbounded(pipeline, View.asMultimap()); }
@Test public void testViewNonmergingAsMultimapDirect() { testViewNonmerging(pipeline, View.asMultimap()); } }
@Test public void testViewNonmergingAsMultimapBatch() { testViewNonmerging(createTestBatchRunner(), View.asMultimap()); }
@Test public void testViewNonmergingAsMultimapStreaming() { testViewNonmerging(createTestStreamingRunner(), View.asMultimap()); } }
@Test public void testViewUnboundedAsMultimapBatch() { testViewUnbounded(createTestBatchRunner(), View.asMultimap()); }
@Test public void testViewUnboundedAsMultimapStreaming() { testViewUnbounded(createTestStreamingRunner(), View.asMultimap()); }
/** * Constructs a {@link SingletonAssert} for the value of the provided {@link PCollection} with the * specified reason. * * <p>Note that the actual value must be coded by a {@link KvCoder}, not just any {@code Coder<K, * V>}. */ public static <K, V> SingletonAssert<Map<K, Iterable<V>>> thatMultimap( String reason, PCollection<KV<K, V>> actual) { @SuppressWarnings("unchecked") KvCoder<K, V> kvCoder = (KvCoder<K, V>) actual.getCoder(); return new PCollectionViewAssert<>( actual, View.asMultimap(), MapCoder.of(kvCoder.getKeyCoder(), IterableCoder.of(kvCoder.getValueCoder())), PAssertionSite.capture(reason)); }
private PCollection<Row> sideInputJoinHelper( JoinRelType joinType, PCollection<KV<Row, Row>> leftRows, PCollection<KV<Row, Row>> rightRows, Row rightNullRow, boolean swapped) { final PCollectionView<Map<Row, Iterable<Row>>> rowsView = rightRows.apply(View.asMultimap()); Schema schema = CalciteUtils.toSchema(getRowType()); PCollection<Row> ret = leftRows .apply( ParDo.of( new BeamJoinTransforms.SideInputJoinDoFn( joinType, rightNullRow, rowsView, swapped, schema)) .withSideInputs(rowsView)) .setRowSchema(schema); return ret; }
@Test public void withViewsForViewNotInContainerFails() { PCollection<KV<String, String>> input = pipeline.apply(Create.empty(new TypeDescriptor<KV<String, String>>() {})); PCollectionView<Map<String, Iterable<String>>> newView = input.apply(View.asMultimap()); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("unknown views"); thrown.expectMessage(newView.toString()); container.createReaderForViews(ImmutableList.of(newView)); }
@Test public void testViewGetName() { assertEquals("View.AsSingleton", View.<Integer>asSingleton().getName()); assertEquals("View.AsIterable", View.<Integer>asIterable().getName()); assertEquals("View.AsMap", View.<String, Integer>asMap().getName()); assertEquals("View.AsMultimap", View.<String, Integer>asMultimap().getName()); }
@Test public void testMultimapSideInputTranslation() throws Exception { assertEquals( ListCoder.of( KvCoder.of(VoidCoder.of(), KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))), getTranslatedSideInputCoder( ImmutableList.of(KV.of("a", 1), KV.of("a", 2), KV.of("b", 3)), View.asMultimap())); }
@Test @Category(ValidatesRunner.class) public void testEmptyMultimapSideInput() throws Exception { final PCollectionView<Map<String, Iterable<Integer>>> view = pipeline .apply( "CreateEmptyView", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))) .apply(View.asMultimap()); PCollection<Integer> results = pipeline .apply("Create1", Create.of(1)) .apply( "OutputSideInputs", ParDo.of( new DoFn<Integer, Integer>() { @ProcessElement public void processElement(ProcessContext c) { assertTrue(c.sideInput(view).isEmpty()); assertTrue(c.sideInput(view).entrySet().isEmpty()); assertFalse(c.sideInput(view).entrySet().iterator().hasNext()); c.output(c.element()); } }) .withSideInputs(view)); // Pass at least one value through to guarantee that DoFn executes. PAssert.that(results).containsInAnyOrder(1); pipeline.run(); }
@Test @Category({ValidatesRunner.class, DataflowPortabilityApiUnsupported.class}) public void testEmptyMultimapSideInputWithNonDeterministicKeyCoder() throws Exception { final PCollectionView<Map<String, Iterable<Integer>>> view = pipeline .apply( "CreateEmptyView", Create.empty(KvCoder.of(new NonDeterministicStringCoder(), VarIntCoder.of()))) .apply(View.asMultimap()); PCollection<Integer> results = pipeline .apply("Create1", Create.of(1)) .apply( "OutputSideInputs", ParDo.of( new DoFn<Integer, Integer>() { @ProcessElement public void processElement(ProcessContext c) { assertTrue(c.sideInput(view).isEmpty()); assertTrue(c.sideInput(view).entrySet().isEmpty()); assertFalse(c.sideInput(view).entrySet().iterator().hasNext()); c.output(c.element()); } }) .withSideInputs(view)); // Pass at least one value through to guarantee that DoFn executes. PAssert.that(results).containsInAnyOrder(1); pipeline.run(); }
pipeline.apply("CreateSideInput", Create.of(KV.of("a", 1))).apply(View.asMultimap());
"CreateSideInput", Create.of(KV.of("a", 1), KV.of("a", 1), KV.of("a", 2), KV.of("b", 3))) .apply(View.asMultimap());
"CreateSideInput", Create.of(KV.of("a", 1), KV.of("a", 1), KV.of("a", 2), KV.of("b", 3))) .apply(View.asMultimap());
Create.of(KV.of("a", 1), KV.of("a", 1), KV.of("a", 2), KV.of("b", 3)) .withCoder(KvCoder.of(new NonDeterministicStringCoder(), VarIntCoder.of()))) .apply(View.asMultimap());
TimestampedValue.of(KV.of("b", 3), new Instant(14)))) .apply("SideWindowInto", Window.into(FixedWindows.of(Duration.millis(10)))) .apply(View.asMultimap());
TimestampedValue.of(KV.of("b", 3), new Instant(14)))) .apply("SideWindowInto", Window.into(FixedWindows.of(Duration.millis(10)))) .apply(View.asMultimap());
.withCoder(KvCoder.of(new NonDeterministicStringCoder(), VarIntCoder.of()))) .apply("SideWindowInto", Window.into(FixedWindows.of(Duration.millis(10)))) .apply(View.asMultimap());