@Test public void testWithFanoutPreservesSideInputs() { final PCollectionView<Integer> view = pipeline.apply(Create.of(1)).apply(Sum.integersGlobally().asSingletonView()); Combine.Globally<Integer, String> combine = Combine.globally(new TestCombineFnWithContext(view)).withSideInputs(view).withFanout(1); assertEquals(Collections.singletonList(view), combine.getSideInputs()); } }
@Test public void testDisplayData() { UniqueInts combineFn = new UniqueInts() { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("fnMetadata", "foobar")); } }; Combine.Globally<?, ?> combine = Combine.globally(combineFn).withFanout(1234); DisplayData displayData = DisplayData.from(combine); assertThat(displayData, hasDisplayItem("combineFn", combineFn.getClass())); assertThat(displayData, hasDisplayItem("emitDefaultOnEmptyInput", true)); assertThat(displayData, hasDisplayItem("fanout", 1234)); assertThat(displayData, includesDisplayDataFor("combineFn", combineFn)); }
@Test @Category(ValidatesRunner.class) public void testHotKeyCombiningWithAccumulationMode() { PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3, 4, 5)); PCollection<Integer> output = input .apply( Window.<Integer>into(new GlobalWindows()) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .accumulatingFiredPanes() .withAllowedLateness(new Duration(0), ClosingBehavior.FIRE_ALWAYS)) .apply(Sum.integersGlobally().withoutDefaults().withFanout(2)) .apply(ParDo.of(new GetLast())); PAssert.that(output) .satisfies( input1 -> { assertThat(input1, hasItem(15)); return null; }); pipeline.run(); }
@Override public PCollectionView<OutputT> expand(PCollection<InputT> input) { PCollection<OutputT> combined = input.apply(Combine.<InputT, OutputT>globally(fn).withoutDefaults().withFanout(fanout)); PCollection<KV<Void, OutputT>> materializationInput = combined.apply(new VoidKeyToMultimapMaterialization<>()); PCollectionView<OutputT> view = PCollectionViews.singletonView( materializationInput, input.getWindowingStrategy(), insertDefault, insertDefault ? fn.defaultValue() : null, combined.getCoder()); materializationInput.apply(CreatePCollectionView.of(view)); return view; }
@Override public PCollection<?> expand(PCollection<T> input) { input = input.apply(Combine.globally(combineFn).withoutDefaults().withFanout(fanout)); @SuppressWarnings("unchecked") Coder<BoundedWindow> windowCoder = (Coder<BoundedWindow>) input.getWindowingStrategy().getWindowFn().windowCoder(); return BatchViewAsSingleton.applyForSingleton( runner, input, new IsmRecordForSingularValuePerWindowDoFn<>(windowCoder), input.getCoder(), view); }