private <T> PTransform<PCollection<T>, PCollection<T>> window() { return Window.into(new GlobalWindows()); }
@Test @Category({NeedsRunner.class, UsesTestStream.class}) public void testWaitInGlobalWindow() { testWaitWithParameters( Duration.standardMinutes(1) /* duration */, Duration.standardSeconds(15) /* lateness */, 20 /* numMainElements */, new GlobalWindows(), 20 /* numSignalElements */, new GlobalWindows()); }
@Test public void testStartBundleExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.startBundle(); }
@Test public void testFinishBundleExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.finishBundle(); }
@Test public void testProcessElementExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.processElement(WindowedValue.valueInGlobalWindow("anyValue")); }
@Override public PCollectionView<Integer> expand(PCollection<T> records) { return records .apply(Window.into(new GlobalWindows())) .apply("CountRecords", Count.globally()) .apply("GenerateShardCount", ParDo.of(new CalculateShardsFn())) .apply(View.asSingleton()); } }
@Test public void classEqualToDoesNotMatchUnrelatedClass() { PTransformMatcher matcher = PTransformMatchers.classEqualTo(ParDo.SingleOutput.class); AppliedPTransform<?, ?, ?> application = getAppliedTransform(Window.<KV<String, Integer>>into(new GlobalWindows())); assertThat(matcher.matches(application), is(false)); }
@Test public void testWindowMappingFnTranslation() throws Exception { SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java")); assertEquals( new GlobalWindows().getDefaultWindowMappingFn(), PCollectionViewTranslation.windowMappingFnFromProto( ParDoTranslation.translateWindowMappingFn( new GlobalWindows().getDefaultWindowMappingFn(), sdkComponents))); }
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testAssignWindowsGlobal() { WindowFn slidingWindows = new GlobalWindows(); WindowAssignTranslator.AssignWindows<String> assignWindows = new WindowAssignTranslator.AssignWindows(slidingWindows); String value = "v1"; Instant timestamp = new Instant(1); WindowedValue<String> windowedValue = WindowedValue.timestampedValueInGlobalWindow(value, timestamp); ArrayList<WindowedValue<String>> expected = new ArrayList<>(); expected.add(WindowedValue.timestampedValueInGlobalWindow(value, timestamp)); Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue); assertThat(expected, equalTo(Lists.newArrayList(result))); } }
@Override public POutput expand(PCollection<? extends T> input) { return input // assign a dummy key and global window, // this is needed to accumulate all observed events in the same state cell .apply(Window.into(new GlobalWindows())) .apply(WithKeys.of("dummyKey")) .apply( "checkAllEventsForSuccess", ParDo.of(new StatefulPredicateCheck<>(coder, formatter, successPredicate))) // signal the success/failure to the result topic .apply("publishSuccess", PubsubIO.writeStrings().to(resultTopicPath.getPath())); } }
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> input) { return input .apply( "LeaderboardUserGlobalWindow", Window.<GameActionInfo>into(new GlobalWindows()) // Get periodic results every ten minutes. .triggering( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(TEN_MINUTES))) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness)) // Extract and sum username/score pairs from the event data. .apply("ExtractUserScore", new ExtractAndSumScore("user")); } }
@Test public void testDisplayDataExcludesDefaults() { Window<?> window = Window.into(new GlobalWindows()) .triggering(DefaultTrigger.of()) .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis())); DisplayData data = DisplayData.from(window); assertThat(data, not(hasDisplayItem("trigger"))); assertThat(data, not(hasDisplayItem("allowedLateness"))); }
@Test public void testDisplayDataExcludesUnspecifiedProperties() { Window<?> onlyHasAccumulationMode = Window.configure().discardingFiredPanes(); assertThat( DisplayData.from(onlyHasAccumulationMode), not( hasDisplayItem( hasKey( isOneOf( "windowFn", "trigger", "timestampCombiner", "allowedLateness", "closingBehavior"))))); Window<?> noAccumulationMode = Window.into(new GlobalWindows()); assertThat( DisplayData.from(noAccumulationMode), not(hasDisplayItem(hasKey("accumulationMode")))); }
@Test public void testUnsupportedGlobalWindowWithDefaultTrigger() { exceptions.expect(UnsupportedOperationException.class); pipeline.enableAbandonedNodeEnforcement(false); PCollection<Row> input = unboundedInput1.apply( "unboundedInput1.globalWindow", Window.<Row>into(new GlobalWindows()).triggering(DefaultTrigger.of())); String sql = "SELECT f_int2, COUNT(*) AS `size` FROM PCOLLECTION GROUP BY f_int2"; input.apply("testUnsupportedGlobalWindows", SqlTransform.query(sql)); }
@Test @Category(ValidatesRunner.class) public void testGlobalCombineWithDefaultsAndTriggers() { PCollection<Integer> input = pipeline.apply(Create.of(1, 1)); PCollection<String> output = input .apply( Window.<Integer>into(new GlobalWindows()) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .accumulatingFiredPanes() .withAllowedLateness(new Duration(0), ClosingBehavior.FIRE_ALWAYS)) .apply(Sum.integersGlobally()) .apply(ParDo.of(new FormatPaneInfo())); // The actual elements produced are nondeterministic. Could be one, could be two. // But it should certainly have a final element with the correct final sum. PAssert.that(output) .satisfies( input1 -> { assertThat(input1, hasItem("2: true")); return null; }); pipeline.run(); }
@Test public void testRepeatedlyElementCount() throws Exception { SimpleTriggerStateMachineTester<GlobalWindow> tester = TriggerStateMachineTester.forTrigger( RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(5)), new GlobalWindows()); GlobalWindow window = GlobalWindow.INSTANCE; tester.injectElements(1); assertFalse(tester.shouldFire(window)); tester.injectElements(2, 3, 4, 5); assertTrue(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertFalse(tester.shouldFire(window)); }
@Parameters(name = "{index}: {0}") public static Iterable<WindowFn<?, ?>> data() { // This pipeline exists for construction, not to run any test. return ImmutableList.<WindowFn<?, ?>>builder() .add(FixedWindows.of(Duration.standardMinutes(10L))) .add(new GlobalWindows()) .add(Sessions.withGapDuration(Duration.standardMinutes(15L))) .add(SlidingWindows.of(Duration.standardMinutes(5L)).every(Duration.standardMinutes(1L))) .add(new CustomWindows()) .build(); }
@Test public void testRepeatedlyAfterFirstElementCount() throws Exception { SimpleTriggerStateMachineTester<GlobalWindow> tester = TriggerStateMachineTester.forTrigger( RepeatedlyStateMachine.forever( AfterFirstStateMachine.of( AfterProcessingTimeStateMachine.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(15)), AfterPaneStateMachine.elementCountAtLeast(5))), new GlobalWindows()); GlobalWindow window = GlobalWindow.INSTANCE; tester.injectElements(1); assertFalse(tester.shouldFire(window)); tester.injectElements(2, 3, 4, 5); assertTrue(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertFalse(tester.shouldFire(window)); }
@Test public void testRepeatedlyProcessingTime() throws Exception { SimpleTriggerStateMachineTester<GlobalWindow> tester = TriggerStateMachineTester.forTrigger( RepeatedlyStateMachine.forever( AfterProcessingTimeStateMachine.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(15))), new GlobalWindows()); GlobalWindow window = GlobalWindow.INSTANCE; tester.injectElements(1); assertFalse(tester.shouldFire(window)); tester.advanceProcessingTime(new Instant(0).plus(Duration.standardMinutes(15))); assertTrue(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertFalse(tester.shouldFire(window)); }
@Test public void testRepeatedlyAfterFirstProcessingTime() throws Exception { SimpleTriggerStateMachineTester<GlobalWindow> tester = TriggerStateMachineTester.forTrigger( RepeatedlyStateMachine.forever( AfterFirstStateMachine.of( AfterProcessingTimeStateMachine.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(15)), AfterPaneStateMachine.elementCountAtLeast(5))), new GlobalWindows()); GlobalWindow window = GlobalWindow.INSTANCE; tester.injectElements(1); assertFalse(tester.shouldFire(window)); tester.advanceProcessingTime(new Instant(0).plus(Duration.standardMinutes(15))); assertTrue(tester.shouldFire(window)); tester.fireIfShouldFire(window); assertFalse(tester.shouldFire(window)); }