@Test public void testFromEndOfWindowToString() { Trigger trigger = AfterWatermark.pastEndOfWindow(); assertEquals("AfterWatermark.pastEndOfWindow()", trigger.toString()); }
@Test public void testTriggerToString() throws Exception { assertEquals("AfterWatermark.pastEndOfWindow()", AfterWatermark.pastEndOfWindow().toString()); assertEquals( "Repeatedly.forever(AfterWatermark.pastEndOfWindow())", Repeatedly.forever(AfterWatermark.pastEndOfWindow()).toString()); }
@Test public void testEarlyFiringsToString() { Trigger trigger = AfterWatermark.pastEndOfWindow().withEarlyFirings(StubTrigger.named("t1")); assertEquals("AfterWatermark.pastEndOfWindow().withEarlyFirings(t1)", trigger.toString()); }
@Test public void testLateFiringsToString() { Trigger trigger = AfterWatermark.pastEndOfWindow().withLateFirings(StubTrigger.named("t1")); assertEquals("AfterWatermark.pastEndOfWindow().withLateFirings(t1)", trigger.toString()); }
@Test public void testEarlyAndLateFiringsToString() { Trigger trigger = AfterWatermark.pastEndOfWindow() .withEarlyFirings(StubTrigger.named("t1")) .withLateFirings(StubTrigger.named("t2")); assertEquals( "AfterWatermark.pastEndOfWindow().withEarlyFirings(t1).withLateFirings(t2)", trigger.toString()); }
@Test public void testToStringExcludesNeverTrigger() { Trigger trigger = AfterWatermark.pastEndOfWindow() .withEarlyFirings(Never.ever()) .withLateFirings(Never.ever()); assertEquals("AfterWatermark.pastEndOfWindow()", trigger.toString()); } }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(TWO_MINUTES) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterAll = AfterAll.of(trigger1, trigger2); assertEquals( AfterAll.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger()), afterAll.getContinuationTrigger()); }
@Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); assertEquals( new Instant(9), AfterEach.inOrder(AfterWatermark.pastEndOfWindow(), AfterPane.elementCountAtLeast(4)) .getWatermarkThatGuaranteesFiring(window)); assertEquals( BoundedWindow.TIMESTAMP_MAX_VALUE, AfterEach.inOrder(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()) .getWatermarkThatGuaranteesFiring(window)); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterFirst = AfterFirst.of(trigger1, trigger2); assertEquals( AfterFirst.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger()), afterFirst.getContinuationTrigger()); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .discardingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testBuiltUpToString() { Trigger trigger = AfterWatermark.pastEndOfWindow() .withLateFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(10))); String expected = "AfterWatermark.pastEndOfWindow()" + ".withLateFirings(AfterProcessingTime" + ".pastFirstElementInPane()" + ".plusDelayOf(10 minutes))"; assertEquals(expected, trigger.toString()); } }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); } }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(Sessions.withGapDuration(ONE_MINUTE)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) { return infos .apply( "LeaderboardTeamFixedWindows", Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration)) // We will get early (speculative) results as well as cumulative // processing of late data. .triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(FIVE_MINUTES)) .withLateFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(TEN_MINUTES))) .withAllowedLateness(allowedLateness) .accumulatingFiredPanes()) // Extract and sum teamname/score pairs from the event data. .apply("ExtractTeamScore", new ExtractAndSumScore("team")); } }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterEach = AfterEach.inOrder(trigger1, trigger2); assertEquals( Repeatedly.forever( AfterFirst.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger())), afterEach.getContinuationTrigger()); }
@Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); assertEquals( BoundedWindow.TIMESTAMP_MAX_VALUE, AfterAll.of(AfterWatermark.pastEndOfWindow(), AfterPane.elementCountAtLeast(1)) .getWatermarkThatGuaranteesFiring(window)); }
public static void main(String[] args) throws Exception { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); options.setStreaming(true); options.setCheckpointingInterval(1000L); options.setNumberOfExecutionRetries(5); options.setExecutionRetryDelay(3000L); options.setRunner(FlinkRunner.class); WindowFn<Object, ?> windowFn = FixedWindows.of( Duration.standardSeconds(options.getWindowSize())); Pipeline p = Pipeline.create(options); // the following two 'applys' create multiple inputs to our pipeline, one for each // of our two input sources. PCollection<String> streamA = p .apply("FirstStream", Read.from(new UnboundedSocketSource<>("localhost", 9999, '\n', 3))) .apply(Window.<String>into(windowFn) .triggering(AfterWatermark.pastEndOfWindow()).withAllowedLateness(Duration.ZERO) .discardingFiredPanes()); PCollection<String> streamB = p .apply("SecondStream", Read.from(new UnboundedSocketSource<>("localhost", 9998, '\n', 3))) .apply(Window.<String>into(windowFn) .triggering(AfterWatermark.pastEndOfWindow()).withAllowedLateness(Duration.ZERO) .discardingFiredPanes()); PCollection<String> formattedResults = joinEvents(streamA, streamB); formattedResults.apply(TextIO.Write.to("./outputJoin.txt")); p.run(); }
/** * With {@link #testWindowIntoWindowFnAssign()}, demonstrates that the expansions of the {@link * Window} transform depends on if it actually assigns elements to windows. */ @Test public void testWindowIntoNullWindowFnNoAssign() { pipeline .apply(Create.of(1, 2, 3)) .apply( Window.<Integer>configure() .triggering(AfterWatermark.pastEndOfWindow()) .withAllowedLateness(Duration.ZERO) .accumulatingFiredPanes()); pipeline.traverseTopologically( new PipelineVisitor.Defaults() { @Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { assertThat(node.getTransform(), not(instanceOf(Window.Assign.class))); } }); }
@Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); assertEquals( new Instant(9), Repeatedly.forever(AfterWatermark.pastEndOfWindow()) .getWatermarkThatGuaranteesFiring(window)); assertEquals( new Instant(9), Repeatedly.forever(AfterWatermark.pastEndOfWindow()) .orFinally(AfterPane.elementCountAtLeast(1)) .getWatermarkThatGuaranteesFiring(window)); assertEquals( new Instant(9), Repeatedly.forever(AfterPane.elementCountAtLeast(1)) .orFinally(AfterWatermark.pastEndOfWindow()) .getWatermarkThatGuaranteesFiring(window)); assertEquals( new Instant(9), AfterPane.elementCountAtLeast(100) .orFinally(AfterWatermark.pastEndOfWindow()) .getWatermarkThatGuaranteesFiring(window)); assertEquals( BoundedWindow.TIMESTAMP_MAX_VALUE, Repeatedly.forever(AfterPane.elementCountAtLeast(1)) .orFinally(AfterPane.elementCountAtLeast(10)) .getWatermarkThatGuaranteesFiring(window)); }