public void advanceInputWatermarkNoTimers(Instant newInputWatermark) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); }
/** * Advance the input watermark to the specified time, then advance the output watermark as far as * possible. */ public void advanceInputWatermark(Instant newInputWatermark) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); while (timerInternals.removeNextEventTimer() != null) { // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 } }
@Override public boolean hasNext() { // Advance if (!hasAdvance) { try { // Finish any pending windows by advancing the input watermark to infinity. timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); // Finally, advance the processing time to infinity to fire any timers. timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); timerInternals.advanceSynchronizedProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); } catch (Exception e) { throw new RuntimeException(e); } hasAdvance = true; } // Get timer data return (timerData = timerInternals.removeNextEventTimer()) != null || (timerData = timerInternals.removeNextProcessingTimer()) != null || (timerData = timerInternals.removeNextSynchronizedProcessingTimer()) != null; }
@Test public void testDeletionById() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); Instant timestamp = new Instant(42); underTest.advanceInputWatermark(new Instant(0)); underTest.setTimer(NS1, ID1, timestamp, TimeDomain.EVENT_TIME); underTest.deleteTimer(NS1, ID1); underTest.advanceInputWatermark(new Instant(43)); assertThat(underTest.removeNextEventTimer(), nullValue()); }
inMemoryTimerInternalsFactory.timerInternalsForKey(key); try { timerInternals.advanceInputWatermark(new Instant(watermark.getTimestamp())); timerInternals.advanceProcessingTime(processingTime); timerInternals.advanceSynchronizedProcessingTime(synchronizedTime);
@Test public void testResetById() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); Instant earlyTimestamp = new Instant(13); Instant laterTimestamp = new Instant(42); underTest.advanceInputWatermark(new Instant(0)); underTest.setTimer(NS1, ID1, earlyTimestamp, TimeDomain.EVENT_TIME); underTest.setTimer(NS1, ID1, laterTimestamp, TimeDomain.EVENT_TIME); underTest.advanceInputWatermark(earlyTimestamp.plus(1L)); assertThat(underTest.removeNextEventTimer(), nullValue()); underTest.advanceInputWatermark(laterTimestamp.plus(1L)); assertThat( underTest.removeNextEventTimer(), equalTo(TimerData.of(ID1, NS1, laterTimestamp, TimeDomain.EVENT_TIME))); }
/** * Advance the input watermark to the specified time, firing any timers that should fire. Then * advance the output watermark as far as possible. */ public void advanceInputWatermark(Instant newInputWatermark) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); while (true) { TimerData timer; List<TimerInternals.TimerData> timers = new ArrayList<>(); while ((timer = timerInternals.removeNextEventTimer()) != null) { timers.add(timer); } if (timers.isEmpty()) { break; } runner.onTimers(timers); } if (autoAdvanceOutputWatermark) { Instant hold = stateInternals.earliestWatermarkHold(); if (hold == null) { WindowTracing.trace( "TestInMemoryTimerInternals.advanceInputWatermark: no holds, " + "so output watermark = input watermark"); hold = timerInternals.currentInputWatermarkTime(); } advanceOutputWatermark(hold); } runner.persist(); }
@Test public void testFiringEventTimers() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTimer1 = TimerData.of(ID1, NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData eventTimer2 = TimerData.of(ID2, NS1, new Instant(29), TimeDomain.EVENT_TIME); underTest.setTimer(eventTimer1); underTest.setTimer(eventTimer2); underTest.advanceInputWatermark(new Instant(20)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer1)); assertThat(underTest.removeNextEventTimer(), nullValue()); // Advancing just a little shouldn't refire underTest.advanceInputWatermark(new Instant(21)); assertThat(underTest.removeNextEventTimer(), nullValue()); // Adding the timer and advancing a little should refire underTest.setTimer(eventTimer1); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer1)); assertThat(underTest.removeNextEventTimer(), nullValue()); // And advancing the rest of the way should still have the other timer underTest.advanceInputWatermark(new Instant(30)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer2)); assertThat(underTest.removeNextEventTimer(), nullValue()); }
timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
private static void advanceInputWatermark( InMemoryTimerInternals timerInternals, Instant newInputWatermark, DoFnRunner<?, ?> toTrigger) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); TimerInternals.TimerData timer; while ((timer = timerInternals.removeNextEventTimer()) != null) { StateNamespace namespace = timer.getNamespace(); checkArgument(namespace instanceof StateNamespaces.WindowNamespace); BoundedWindow window = ((StateNamespaces.WindowNamespace) namespace).getWindow(); toTrigger.onTimer(timer.getTimerId(), window, timer.getTimestamp(), timer.getDomain()); } }
timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
@Test public void testDeduplicate() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTime = TimerData.of(NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData processingTime = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); underTest.setTimer(eventTime); underTest.setTimer(eventTime); underTest.setTimer(processingTime); underTest.setTimer(processingTime); underTest.advanceProcessingTime(new Instant(20)); underTest.advanceInputWatermark(new Instant(20)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime)); assertThat(underTest.removeNextEventTimer(), nullValue()); } }
timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
underTest.advanceInputWatermark(new Instant(30)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime1)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime2));
timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
@Test public void testLateDropping() throws Exception { MetricsContainerImpl container = new MetricsContainerImpl("any"); MetricsEnvironment.setCurrentContainer(container); timerInternals.advanceInputWatermark(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE)); timerInternals.advanceOutputWatermark(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE)); DoFn<KV<String, Integer>, Integer> fn = new MyDoFn(); DoFnRunner<KV<String, Integer>, Integer> runner = DoFnRunners.defaultStatefulDoFnRunner( fn, getDoFnRunner(fn), WINDOWING_STRATEGY, new StatefulDoFnRunner.TimeInternalsCleanupTimer(timerInternals, WINDOWING_STRATEGY), new StatefulDoFnRunner.StateInternalsStateCleaner<>( fn, stateInternals, (Coder) WINDOWING_STRATEGY.getWindowFn().windowCoder())); runner.startBundle(); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(0L + WINDOW_SIZE)); Instant timestamp = new Instant(0); runner.processElement( WindowedValue.of(KV.of("hello", 1), timestamp, window, PaneInfo.NO_FIRING)); long droppedValues = container .getCounter( MetricName.named( StatefulDoFnRunner.class, StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)) .getCumulative(); assertEquals(1L, droppedValues); runner.finishBundle(); }
@Test public void testGarbageCollect() throws Exception { timerInternals.advanceInputWatermark(new Instant(1L));