@Override public boolean hasNext() { // Advance if (!hasAdvance) { try { // Finish any pending windows by advancing the input watermark to infinity. timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); // Finally, advance the processing time to infinity to fire any timers. timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); timerInternals.advanceSynchronizedProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); } catch (Exception e) { throw new RuntimeException(e); } hasAdvance = true; } // Get timer data return (timerData = timerInternals.removeNextEventTimer()) != null || (timerData = timerInternals.removeNextProcessingTimer()) != null || (timerData = timerInternals.removeNextSynchronizedProcessingTimer()) != null; }
@Test public void testDeletionIdempotent() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); Instant timestamp = new Instant(42); underTest.setTimer(NS1, ID1, timestamp, TimeDomain.EVENT_TIME); underTest.deleteTimer(NS1, ID1); underTest.deleteTimer(NS1, ID1); }
/** Advance the processing time to the specified time. */ public void advanceProcessingTime(Instant newProcessingTime) throws Exception { timerInternals.advanceProcessingTime(newProcessingTime); while (timerInternals.removeNextProcessingTimer() != null) { // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 } timerInternals.advanceSynchronizedProcessingTime(newProcessingTime); while (timerInternals.removeNextSynchronizedProcessingTimer() != null) { // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 } }
/** * Get timer data. * @param timerInternals in-memory timer internals. * @return list of timer datas. */ private List<TimerInternals.TimerData> getEligibleTimers(final InMemoryTimerInternals timerInternals) { final List<TimerInternals.TimerData> timerData = new LinkedList<>(); while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; timerData.add(timer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; timerData.add(timer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; timerData.add(timer); } if (!hasFired) { break; } } return timerData; }
final InMemoryTimerInternals timerInternals = new InMemoryTimerInternals(); timerInternals.advanceProcessingTime(Instant.now()); timerInternals.advanceSynchronizedProcessingTime(Instant.now()); (WindowedValue timerElement, TimerInternals.TimerData timerData) -> { currentTimerKey = ((KV) timerElement.getValue()).getKey(); timerInternals.setTimer(timerData); }, windowCoder)); timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); timerInternals.advanceSynchronizedProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE);
@Test public void testTimerOrdering() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTime1 = TimerData.of(NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData processingTime1 = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); underTest.setTimer(processingTime1); underTest.setTimer(eventTime1); underTest.setTimer(synchronizedProcessingTime1); underTest.setTimer(processingTime2); underTest.setTimer(eventTime2); underTest.setTimer(synchronizedProcessingTime2); assertThat(underTest.removeNextEventTimer(), nullValue()); underTest.advanceInputWatermark(new Instant(30)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime1)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime2)); assertThat(underTest.removeNextEventTimer(), nullValue()); assertThat(underTest.removeNextProcessingTimer(), nullValue()); underTest.advanceProcessingTime(new Instant(30)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime1)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime2)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); assertThat(underTest.removeNextSynchronizedProcessingTimer(), nullValue()); underTest.advanceSynchronizedProcessingTime(new Instant(30)); assertThat( underTest.removeNextSynchronizedProcessingTimer(), equalTo(synchronizedProcessingTime1)); assertThat(
@Test public void testDeletionById() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); Instant timestamp = new Instant(42); underTest.advanceInputWatermark(new Instant(0)); underTest.setTimer(NS1, ID1, timestamp, TimeDomain.EVENT_TIME); underTest.deleteTimer(NS1, ID1); underTest.advanceInputWatermark(new Instant(43)); assertThat(underTest.removeNextEventTimer(), nullValue()); }
@Test public void testDeduplicate() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTime = TimerData.of(NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData processingTime = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); underTest.setTimer(eventTime); underTest.setTimer(eventTime); underTest.setTimer(processingTime); underTest.setTimer(processingTime); underTest.advanceProcessingTime(new Instant(20)); underTest.advanceInputWatermark(new Instant(20)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime)); assertThat(underTest.removeNextEventTimer(), nullValue()); } }
@Test public void testFiringProcessingTimeTimers() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData processingTime1 = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); TimerData processingTime2 = TimerData.of(NS1, new Instant(29), TimeDomain.PROCESSING_TIME); underTest.setTimer(processingTime1); underTest.setTimer(processingTime2); underTest.advanceProcessingTime(new Instant(20)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime1)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // Advancing just a little shouldn't refire underTest.advanceProcessingTime(new Instant(21)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // Adding the timer and advancing a little should fire again underTest.setTimer(processingTime1); underTest.advanceProcessingTime(new Instant(21)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime1)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // And advancing the rest of the way should still have the other timer underTest.advanceProcessingTime(new Instant(30)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime2)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); }
inMemoryTimerInternalsFactory.timerInternalsForKey(key); try { timerInternals.advanceInputWatermark(new Instant(watermark.getTimestamp())); timerInternals.advanceProcessingTime(processingTime); timerInternals.advanceSynchronizedProcessingTime(synchronizedTime); } catch (final Exception e) { throw new RuntimeException(e);
/** * Advance the input watermark to the specified time, then advance the output watermark as far as * possible. */ public void advanceInputWatermark(Instant newInputWatermark) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); while (timerInternals.removeNextEventTimer() != null) { // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 } }
/** * Advances processing time by a given duration and, if any timers fired, performs a non-seed * {@link DoFn.ProcessElement} call, feeding it the timers. */ boolean advanceProcessingTimeBy(Duration duration) throws Exception { currentProcessingTime = currentProcessingTime.plus(duration); timerInternals.advanceProcessingTime(currentProcessingTime); List<TimerInternals.TimerData> timers = new ArrayList<>(); TimerInternals.TimerData nextTimer; while ((nextTimer = timerInternals.removeNextProcessingTimer()) != null) { timers.add(nextTimer); } if (timers.isEmpty()) { return false; } tester.processElement( KeyedWorkItems.timersWorkItem("key".getBytes(StandardCharsets.UTF_8), timers)); return true; }
/** * Advance the input watermark to the specified time, firing any timers that should fire. Then * advance the output watermark as far as possible. */ public void advanceInputWatermark(Instant newInputWatermark) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); while (true) { TimerData timer; List<TimerInternals.TimerData> timers = new ArrayList<>(); while ((timer = timerInternals.removeNextEventTimer()) != null) { timers.add(timer); } if (timers.isEmpty()) { break; } runner.onTimers(timers); } if (autoAdvanceOutputWatermark) { Instant hold = stateInternals.earliestWatermarkHold(); if (hold == null) { WindowTracing.trace( "TestInMemoryTimerInternals.advanceInputWatermark: no holds, " + "so output watermark = input watermark"); hold = timerInternals.currentInputWatermarkTime(); } advanceOutputWatermark(hold); } runner.persist(); }
/** * Advance the synchronized processing time to the specified time, firing any timers that should * fire. */ public void advanceSynchronizedProcessingTime(Instant newSynchronizedProcessingTime) throws Exception { timerInternals.advanceSynchronizedProcessingTime(newSynchronizedProcessingTime); ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); while (true) { TimerData timer; List<TimerInternals.TimerData> timers = new ArrayList<>(); while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { timers.add(timer); } if (timers.isEmpty()) { break; } runner.onTimers(timers); } runner.persist(); }
new ProcessFn<>(fn, inputCoder, restrictionCoder, windowingStrategy); this.tester = DoFnTester.of(processFn); this.timerInternals = new InMemoryTimerInternals(); this.stateInternals = new TestInMemoryStateInternals<>("dummy"); processFn.setStateInternalsFactory(key -> stateInternals); timerInternals.advanceProcessingTime(currentProcessingTime);
public void advanceInputWatermarkNoTimers(Instant newInputWatermark) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); }
public void advanceProcessingTimeNoTimers(Instant newProcessingTime) throws Exception { timerInternals.advanceProcessingTime(newProcessingTime); }
@Before public void setup() { MockitoAnnotations.initMocks(this); when(mockStepContext.timerInternals()).thenReturn(timerInternals); stateInternals = new InMemoryStateInternals<>("hello"); timerInternals = new InMemoryTimerInternals(); when(mockStepContext.stateInternals()).thenReturn((StateInternals) stateInternals); when(mockStepContext.timerInternals()).thenReturn(timerInternals); }
@Test public void testLateDropping() throws Exception { MetricsContainerImpl container = new MetricsContainerImpl("any"); MetricsEnvironment.setCurrentContainer(container); timerInternals.advanceInputWatermark(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE)); timerInternals.advanceOutputWatermark(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE)); DoFn<KV<String, Integer>, Integer> fn = new MyDoFn(); DoFnRunner<KV<String, Integer>, Integer> runner = DoFnRunners.defaultStatefulDoFnRunner( fn, getDoFnRunner(fn), WINDOWING_STRATEGY, new StatefulDoFnRunner.TimeInternalsCleanupTimer(timerInternals, WINDOWING_STRATEGY), new StatefulDoFnRunner.StateInternalsStateCleaner<>( fn, stateInternals, (Coder) WINDOWING_STRATEGY.getWindowFn().windowCoder())); runner.startBundle(); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(0L + WINDOW_SIZE)); Instant timestamp = new Instant(0); runner.processElement( WindowedValue.of(KV.of("hello", 1), timestamp, window, PaneInfo.NO_FIRING)); long droppedValues = container .getCounter( MetricName.named( StatefulDoFnRunner.class, StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)) .getCumulative(); assertEquals(1L, droppedValues); runner.finishBundle(); }
@Override public void setTimer( StateNamespace namespace, String timerId, Instant target, TimeDomain timeDomain) { setTimer(TimerData.of(timerId, namespace, target, timeDomain)); }