@Override protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) { // Use OrFinallyTrigger instead of AfterFirst because the continuation of ACTUAL // may not be a OnceTrigger. return Repeatedly.forever( new OrFinallyTrigger( continuationTriggers.get(ACTUAL), (Trigger.OnceTrigger) continuationTriggers.get(UNTIL))); }
/** * Create a composite trigger that repeatedly executes the trigger {@code repeated}, firing each * time it fires and ignoring any indications to finish. * * <p>Unless used with {@link Trigger#orFinally} the composite trigger will never finish. * * @param repeated the trigger to execute repeatedly. */ public static Repeatedly forever(Trigger repeated) { return new Repeatedly(repeated); }
private RunnerApi.Trigger convertSpecific(Repeatedly v) { return RunnerApi.Trigger.newBuilder() .setRepeat( RunnerApi.Trigger.Repeat.newBuilder().setSubtrigger(toProto(v.getRepeatedTrigger()))) .build(); }
/** Tests that the watermark that guarantees firing is that of the subtrigger. */ @Test public void testFireDeadline() throws Exception { setUp(FixedWindows.of(Duration.millis(10))); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); Instant arbitraryInstant = new Instant(34957849); when(mockTrigger.getWatermarkThatGuaranteesFiring(Mockito.<IntervalWindow>any())) .thenReturn(arbitraryInstant); assertThat( Repeatedly.forever(mockTrigger).getWatermarkThatGuaranteesFiring(window), equalTo(arbitraryInstant)); }
.triggering(Repeatedly.forever(AfterWatermark.pastEndOfWindow())) FixedWindows.of(Duration.standardMinutes(windowDuration))) .triggering(Repeatedly.forever(AfterWatermark.pastEndOfWindow())) Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane() .triggering( AfterEach.inOrder( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane() .orFinally(AfterWatermark.pastEndOfWindow()), Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane()
@Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); assertEquals( new Instant(9), Repeatedly.forever(AfterWatermark.pastEndOfWindow()) .getWatermarkThatGuaranteesFiring(window)); assertEquals( new Instant(9), Repeatedly.forever(AfterWatermark.pastEndOfWindow()) .orFinally(AfterPane.elementCountAtLeast(1)) .getWatermarkThatGuaranteesFiring(window)); assertEquals( new Instant(9), Repeatedly.forever(AfterPane.elementCountAtLeast(1)) .orFinally(AfterWatermark.pastEndOfWindow()) .getWatermarkThatGuaranteesFiring(window)); assertEquals( new Instant(9), AfterPane.elementCountAtLeast(100) .orFinally(AfterWatermark.pastEndOfWindow()) .getWatermarkThatGuaranteesFiring(window)); assertEquals( BoundedWindow.TIMESTAMP_MAX_VALUE, Repeatedly.forever(AfterPane.elementCountAtLeast(1)) .orFinally(AfterPane.elementCountAtLeast(10)) .getWatermarkThatGuaranteesFiring(window)); }
@Override protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) { return Repeatedly.forever(new AfterFirst(continuationTriggers)); }
@Override protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) { return new Repeatedly(continuationTriggers.get(REPEATED)); }
@Test public void testToString() { Trigger trigger = Repeatedly.forever( new StubTrigger() { @Override public String toString() { return "innerTrigger"; } }); assertEquals("Repeatedly.forever(innerTrigger)", trigger.toString()); } }
@Test public void testContinuation() throws Exception { Trigger trigger = AfterProcessingTime.pastFirstElementInPane(); Trigger repeatedly = Repeatedly.forever(trigger); assertEquals( Repeatedly.forever(trigger.getContinuationTrigger()), repeatedly.getContinuationTrigger()); assertEquals( Repeatedly.forever(trigger.getContinuationTrigger().getContinuationTrigger()), repeatedly.getContinuationTrigger().getContinuationTrigger()); }
@Test public void testPaneInfoFinalAndOnTime() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever(AfterPane.elementCountAtLeast(2)) .orFinally(AfterWatermark.pastEndOfWindow())) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)); tester.advanceInputWatermark(new Instant(0)); // Should trigger due to element count tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); assertThat( tester.extractOutput(), contains( WindowMatchers.valueWithPaneInfo( PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)))); tester.advanceInputWatermark(new Instant(150)); assertThat( tester.extractOutput(), contains( WindowMatchers.valueWithPaneInfo( PaneInfo.createPane(false, true, Timing.ON_TIME, 1, 0)))); }
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> input) { return input .apply( "LeaderboardUserGlobalWindow", Window.<GameActionInfo>into(new GlobalWindows()) // Get periodic results every ten minutes. .triggering( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(TEN_MINUTES))) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness)) // Extract and sum username/score pairs from the event data. .apply("ExtractUserScore", new ExtractAndSumScore("user")); } }
@Override public PCollection<Void> expand(PCollection<KV<K, V>> input) { int numShards = spec.getNumShards(); if (numShards <= 0) { try (Consumer<?, ?> consumer = openConsumer(spec)) { numShards = consumer.partitionsFor(spec.getTopic()).size(); LOG.info( "Using {} shards for exactly-once writer, matching number of partitions " + "for topic '{}'", numShards, spec.getTopic()); } } checkState(numShards > 0, "Could not set number of shards"); return input .apply( Window.<KV<K, V>>into(new GlobalWindows()) // Everything into global window. .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .discardingFiredPanes()) .apply( String.format("Shuffle across %d shards", numShards), ParDo.of(new Reshard<>(numShards))) .apply("Persist sharding", GroupByKey.create()) .apply("Assign sequential ids", ParDo.of(new Sequencer<>())) .apply("Persist ids", GroupByKey.create()) .apply( String.format("Write to Kafka topic '%s'", spec.getTopic()), ParDo.of(new ExactlyOnceWriter<>(spec, input.getCoder()))); }
AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(42))) .withLateFirings(AfterPane.elementCountAtLeast(3))), toProtoAndBackSpec(Repeatedly.forever(AfterWatermark.pastEndOfWindow())), toProtoAndBackSpec( Repeatedly.forever(AfterPane.elementCountAtLeast(1)) .orFinally(AfterWatermark.pastEndOfWindow())));
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }