@Test public void testToString() { Trigger trigger = AfterProcessingTime.pastFirstElementInPane(); assertEquals("AfterProcessingTime.pastFirstElementInPane()", trigger.toString()); }
/** Basic test of compatibility check between identical triggers. */ @Test public void testCompatibilityIdentical() throws Exception { Trigger t1 = AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardMinutes(1L)); Trigger t2 = AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardMinutes(1L)); assertTrue(t1.isCompatible(t2)); }
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) { return infos .apply( "LeaderboardTeamFixedWindows", Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration)) // We will get early (speculative) results as well as cumulative // processing of late data. .triggering( AfterWatermark.pastEndOfWindow() .withEarlyFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(FIVE_MINUTES)) .withLateFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(TEN_MINUTES))) .withAllowedLateness(allowedLateness) .accumulatingFiredPanes()) // Extract and sum teamname/score pairs from the event data. .apply("ExtractTeamScore", new ExtractAndSumScore("team")); } }
@Test public void testWithDelayToString() { Trigger trigger = AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardMinutes(5)); assertEquals( "AfterProcessingTime.pastFirstElementInPane().plusDelayOf(5 minutes)", trigger.toString()); }
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> input) { return input .apply( "LeaderboardUserGlobalWindow", Window.<GameActionInfo>into(new GlobalWindows()) // Get periodic results every ten minutes. .triggering( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(TEN_MINUTES))) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness)) // Extract and sum username/score pairs from the event data. .apply("ExtractUserScore", new ExtractAndSumScore("user")); } }
@Test public void testContinuation() throws Exception { OnceTrigger firstElementPlus1 = AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardHours(1)); assertEquals( AfterSynchronizedProcessingTime.ofFirstElement(), firstElementPlus1.getContinuationTrigger()); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(TWO_MINUTES))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(Repeatedly.forever(AfterProcessingTime .pastFirstElementInPane() .alignedTo(TWO_MINUTES, Utils.parseTime("12:05:00")))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(TWO_MINUTES) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterAll = AfterAll.of(trigger1, trigger2); assertEquals( AfterAll.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger()), afterAll.getContinuationTrigger()); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterFirst = AfterFirst.of(trigger1, trigger2); assertEquals( AfterFirst.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger()), afterFirst.getContinuationTrigger()); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .discardingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testBuiltUpToString() { Trigger trigger = AfterWatermark.pastEndOfWindow() .withLateFirings( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.standardMinutes(10))); String expected = "AfterWatermark.pastEndOfWindow()" + ".withLateFirings(AfterProcessingTime" + ".pastFirstElementInPane()" + ".plusDelayOf(10 minutes))"; assertEquals(expected, trigger.toString()); } }
@Test public void testFireDeadline() throws Exception { assertEquals( BoundedWindow.TIMESTAMP_MAX_VALUE, AfterProcessingTime.pastFirstElementInPane() .getWatermarkThatGuaranteesFiring(new IntervalWindow(new Instant(0), new Instant(10)))); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); } }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(Sessions.withGapDuration(ONE_MINUTE)) .triggering(AfterWatermark.pastEndOfWindow() .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(ONE_MINUTE)) .withLateFirings(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterEach = AfterEach.inOrder(trigger1, trigger2); assertEquals( Repeatedly.forever( AfterFirst.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger())), afterEach.getContinuationTrigger()); }
@Test public void testContinuation() throws Exception { Trigger trigger = AfterProcessingTime.pastFirstElementInPane(); Trigger repeatedly = Repeatedly.forever(trigger); assertEquals( Repeatedly.forever(trigger.getContinuationTrigger()), repeatedly.getContinuationTrigger()); assertEquals( Repeatedly.forever(trigger.getContinuationTrigger().getContinuationTrigger()), repeatedly.getContinuationTrigger().getContinuationTrigger()); }
@Test public void testContinuation() throws Exception { OnceTrigger triggerA = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger triggerB = AfterWatermark.pastEndOfWindow(); Trigger aOrFinallyB = triggerA.orFinally(triggerB); Trigger bOrFinallyA = triggerB.orFinally(triggerA); assertEquals( Repeatedly.forever( triggerA.getContinuationTrigger().orFinally(triggerB.getContinuationTrigger())), aOrFinallyB.getContinuationTrigger()); assertEquals( Repeatedly.forever( triggerB.getContinuationTrigger().orFinally(triggerA.getContinuationTrigger())), bOrFinallyA.getContinuationTrigger()); }
public final static void main(String args[]) throws Exception { final Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline pipeline = Pipeline.create(options); pipeline .apply(KafkaIO.<Long, String>read() .withBootstrapServers(options.getBootstrap()) .withTopic(options.getTopic()) .withKeyDeserializer(LongDeserializer.class) .withValueDeserializer(StringDeserializer.class) .withoutMetadata()) .apply(Values.<String>create()) .apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(10))) .triggering(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardSeconds(10)))) .withAllowedLateness(Duration.ZERO) .discardingFiredPanes() ) .apply(TextIO.write() .to(options.getOutput()) .withWindowedWrites() .withNumShards(1)); pipeline.run().waitUntilFinish(); } }