@Test public void testFromEndOfWindowToString() { Trigger trigger = AfterWatermark.pastEndOfWindow(); assertEquals("AfterWatermark.pastEndOfWindow()", trigger.toString()); }
@Test public void testTriggerToString() throws Exception { assertEquals("AfterWatermark.pastEndOfWindow()", AfterWatermark.pastEndOfWindow().toString()); assertEquals( "Repeatedly.forever(AfterWatermark.pastEndOfWindow())", Repeatedly.forever(AfterWatermark.pastEndOfWindow()).toString()); }
@Test public void testEarlyFiringsToString() { Trigger trigger = AfterWatermark.pastEndOfWindow().withEarlyFirings(StubTrigger.named("t1")); assertEquals("AfterWatermark.pastEndOfWindow().withEarlyFirings(t1)", trigger.toString()); }
@Test public void testLateFiringsToString() { Trigger trigger = AfterWatermark.pastEndOfWindow().withLateFirings(StubTrigger.named("t1")); assertEquals("AfterWatermark.pastEndOfWindow().withLateFirings(t1)", trigger.toString()); }
@Test public void testEarlyAndLateFiringsToString() { Trigger trigger = AfterWatermark.pastEndOfWindow() .withEarlyFirings(StubTrigger.named("t1")) .withLateFirings(StubTrigger.named("t2")); assertEquals( "AfterWatermark.pastEndOfWindow().withEarlyFirings(t1).withLateFirings(t2)", trigger.toString()); }
@Test public void testToStringExcludesNeverTrigger() { Trigger trigger = AfterWatermark.pastEndOfWindow() .withEarlyFirings(Never.ever()) .withLateFirings(Never.ever()); assertEquals("AfterWatermark.pastEndOfWindow()", trigger.toString()); } }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(TWO_MINUTES) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterAll = AfterAll.of(trigger1, trigger2); assertEquals( AfterAll.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger()), afterAll.getContinuationTrigger()); }
@Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); assertEquals( new Instant(9), AfterEach.inOrder(AfterWatermark.pastEndOfWindow(), AfterPane.elementCountAtLeast(4)) .getWatermarkThatGuaranteesFiring(window)); assertEquals( BoundedWindow.TIMESTAMP_MAX_VALUE, AfterEach.inOrder(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()) .getWatermarkThatGuaranteesFiring(window)); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterFirst = AfterFirst.of(trigger1, trigger2); assertEquals( AfterFirst.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger()), afterFirst.getContinuationTrigger()); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .discardingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testBuiltUpToString() { Trigger trigger = AfterWatermark.pastEndOfWindow() .withLateFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(10))); String expected = "AfterWatermark.pastEndOfWindow()" + ".withLateFirings(AfterProcessingTime" + ".pastFirstElementInPane()" + ".plusDelayOf(10 minutes))"; assertEquals(expected, trigger.toString()); } }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); } }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(Sessions.withGapDuration(ONE_MINUTE)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterEach = AfterEach.inOrder(trigger1, trigger2); assertEquals( Repeatedly.forever( AfterFirst.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger())), afterEach.getContinuationTrigger()); }
@Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); assertEquals( BoundedWindow.TIMESTAMP_MAX_VALUE, AfterAll.of(AfterWatermark.pastEndOfWindow(), AfterPane.elementCountAtLeast(1)) .getWatermarkThatGuaranteesFiring(window)); }
/** * With {@link #testWindowIntoWindowFnAssign()}, demonstrates that the expansions of the {@link * Window} transform depends on if it actually assigns elements to windows. */ @Test public void testWindowIntoNullWindowFnNoAssign() { pipeline .apply(Create.of(1, 2, 3)) .apply( Window.<Integer>configure() .triggering(AfterWatermark.pastEndOfWindow()) .withAllowedLateness(Duration.ZERO) .accumulatingFiredPanes()); pipeline.traverseTopologically( new PipelineVisitor.Defaults() { @Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { assertThat(node.getTransform(), not(instanceOf(Window.Assign.class))); } }); }
public void logNRows(PFeatureRows pFeatureRows, String name, int limit) { PCollection<FeatureRowExtended> main = pFeatureRows.getMain(); PCollection<FeatureRowExtended> errors = pFeatureRows.getErrors(); if (main.isBounded().equals(IsBounded.UNBOUNDED)) { Window<FeatureRowExtended> minuteWindow = Window.<FeatureRowExtended>into(FixedWindows.of(Duration.standardMinutes(1L))) .triggering(AfterWatermark.pastEndOfWindow()) .discardingFiredPanes() .withAllowedLateness(Duration.standardMinutes(1)); main = main.apply(minuteWindow); errors = errors.apply(minuteWindow); } main.apply("Sample success", Sample.any(limit)) .apply("Log success sample", ParDo.of(new LoggerDoFn(Level.INFO, name + " MAIN "))); errors .apply("Sample errors", Sample.any(limit)) .apply("Log errors sample", ParDo.of(new LoggerDoFn(Level.ERROR, name + " ERRORS "))); }
@Test public void testFireDeadline() throws Exception { BoundedWindow window = new IntervalWindow(new Instant(0), new Instant(10)); assertEquals( new Instant(9), AfterFirst.of(AfterWatermark.pastEndOfWindow(), AfterPane.elementCountAtLeast(4)) .getWatermarkThatGuaranteesFiring(window)); assertEquals( BoundedWindow.TIMESTAMP_MAX_VALUE, AfterFirst.of(AfterPane.elementCountAtLeast(2), AfterPane.elementCountAtLeast(1)) .getWatermarkThatGuaranteesFiring(window)); }
@Test public void testContinuation() throws Exception { OnceTrigger triggerA = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger triggerB = AfterWatermark.pastEndOfWindow(); Trigger aOrFinallyB = triggerA.orFinally(triggerB); Trigger bOrFinallyA = triggerB.orFinally(triggerA); assertEquals( Repeatedly.forever( triggerA.getContinuationTrigger().orFinally(triggerB.getContinuationTrigger())), aOrFinallyB.getContinuationTrigger()); assertEquals( Repeatedly.forever( triggerB.getContinuationTrigger().orFinally(triggerA.getContinuationTrigger())), bOrFinallyA.getContinuationTrigger()); }