@Override public PCollection<String> expand(PCollection<String> in) { return in.apply( "Window", Window.<String>into(windowFn).withTimestampCombiner(TimestampCombiner.EARLIEST)) .apply(Count.perElement()) .apply("FormatCounts", ParDo.of(new FormatCountsDoFn())) .setCoder(StringUtf8Coder.of()); } }
"WindowClicks", Window.<KV<Integer, String>>into(FixedWindows.of(new Duration(4))) .withTimestampCombiner(TimestampCombiner.EARLIEST)); "WindowPurchases", Window.<KV<Integer, String>>into(FixedWindows.of(new Duration(4))) .withTimestampCombiner(TimestampCombiner.EARLIEST));
.discardingFiredPanes() .withAllowedLateness(inputWindowingStrategy.getAllowedLateness()) .withTimestampCombiner(TimestampCombiner.EARLIEST))
.triggering(new ReshuffleTrigger<>()) .discardingFiredPanes() .withTimestampCombiner(TimestampCombiner.EARLIEST) .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
@Test public void testDisplayData() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow(); Duration allowedLateness = Duration.standardMinutes(10); Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY; TimestampCombiner timestampCombiner = TimestampCombiner.END_OF_WINDOW; Window<?> window = Window.into(windowFn) .triggering(triggerBuilder) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness, closingBehavior) .withTimestampCombiner(timestampCombiner); DisplayData displayData = DisplayData.from(window); assertThat(displayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(displayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(displayData, hasDisplayItem("trigger", triggerBuilder.toString())); assertThat( displayData, hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString())); assertThat(displayData, hasDisplayItem("allowedLateness", allowedLateness)); assertThat(displayData, hasDisplayItem("closingBehavior", closingBehavior.toString())); assertThat(displayData, hasDisplayItem("timestampCombiner", timestampCombiner.toString())); }
/** * Tests that when two elements are combined via a GroupByKey their output timestamp agrees with * the windowing function customized to use the latest value. */ @Test @Category(ValidatesRunner.class) public void testTimestampCombinerLatest() { p.apply( Create.timestamped( TimestampedValue.of(KV.of(0, "hello"), new Instant(0)), TimestampedValue.of(KV.of(0, "goodbye"), new Instant(10)))) .apply( Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(10))) .withTimestampCombiner(TimestampCombiner.LATEST)) .apply(GroupByKey.create()) .apply(ParDo.of(new AssertTimestamp(new Instant(10)))); p.run(); }
/** * Tests that when two elements are combined via a GroupByKey their output timestamp agrees with * the windowing function customized to actually be the same as the default, the earlier of the * two values. */ @Test @Category(ValidatesRunner.class) public void testTimestampCombinerEarliest() { p.apply( Create.timestamped( TimestampedValue.of(KV.of(0, "hello"), new Instant(0)), TimestampedValue.of(KV.of(0, "goodbye"), new Instant(10)))) .apply( Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(10))) .withTimestampCombiner(TimestampCombiner.EARLIEST)) .apply(GroupByKey.create()) .apply(ParDo.of(new AssertTimestamp(new Instant(0)))); p.run(); }
.apply( Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))) .withTimestampCombiner(TimestampCombiner.LATEST)) .apply(Count.perElement()) .apply(ParDo.of(new KeyedByTimestamp<>()))
/** * Tests that when two elements are combined via a GroupByKey their output timestamp agrees with * the windowing function customized to use the end of the window. */ @Test @Category(ValidatesRunner.class) public void testTimestampCombinerEndOfWindow() { pipeline.enableAbandonedNodeEnforcement(true); pipeline .apply( Create.timestamped( TimestampedValue.of(KV.of(0, "hello"), new Instant(0)), TimestampedValue.of(KV.of(0, "goodbye"), new Instant(10)))) .apply( Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(10))) .withTimestampCombiner(TimestampCombiner.END_OF_WINDOW)) .apply(GroupByKey.create()) .apply( ParDo.of( new DoFn<KV<Integer, Iterable<String>>, Void>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { assertThat(c.timestamp(), equalTo(new Instant(10 * 60 * 1000 - 1))); } })); pipeline.run(); }
@Test @Category(ValidatesRunner.class) public void testPrimitiveDisplayData() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow(); Duration allowedLateness = Duration.standardMinutes(10); Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY; TimestampCombiner timestampCombiner = TimestampCombiner.END_OF_WINDOW; Window<?> window = Window.into(windowFn) .triggering(triggerBuilder) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness, closingBehavior) .withTimestampCombiner(timestampCombiner); DisplayData primitiveDisplayData = Iterables.getOnlyElement( DisplayDataEvaluator.create().displayDataForPrimitiveTransforms(window)); assertThat(primitiveDisplayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(primitiveDisplayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(primitiveDisplayData, hasDisplayItem("trigger", triggerBuilder.toString())); assertThat( primitiveDisplayData, hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString())); assertThat(primitiveDisplayData, hasDisplayItem("allowedLateness", allowedLateness)); assertThat(primitiveDisplayData, hasDisplayItem("closingBehavior", closingBehavior.toString())); assertThat( primitiveDisplayData, hasDisplayItem("timestampCombiner", timestampCombiner.toString())); }
/** * Tests that when a processing time timers comes in after a window is expired it does not cause * a spurious output. */ @Test @Category({ValidatesRunner.class, UsesTestStream.class}) public void testCombiningAccumulatingProcessingTime() throws Exception { PCollection<Integer> triggeredSums = p.apply( TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(new Instant(0)) .addElements( TimestampedValue.of(2, new Instant(2)), TimestampedValue.of(5, new Instant(5))) .advanceWatermarkTo(new Instant(100)) .advanceProcessingTime(Duration.millis(10)) .advanceWatermarkToInfinity()) .apply( Window.<Integer>into(FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .accumulatingFiredPanes() .withAllowedLateness(Duration.ZERO) .triggering( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.millis(10))))) .apply(Sum.integersGlobally().withoutDefaults()); PAssert.that(triggeredSums).containsInAnyOrder(7); p.run(); }
.apply( Window.<KV<String, Long>>into(FixedWindows.of(Duration.millis(5L))) .withTimestampCombiner(combiner)) .apply(Combine.perKey(new MultiStepCombineFn())); PCollection<KV<String, TimestampedValue<Long>>> reified =
Window.<KV<String, Integer>>into( Sessions.withGapDuration(Duration.standardMinutes(options.getSessionGap()))) .withTimestampCombiner(TimestampCombiner.END_OF_WINDOW))