private void fireEligibleTimers( InMemoryTimerInternals timerInternals, ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner) throws Exception { List<TimerInternals.TimerData> timers = new ArrayList<>(); while (true) { TimerInternals.TimerData timer; while ((timer = timerInternals.removeNextEventTimer()) != null) { timers.add(timer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { timers.add(timer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { timers.add(timer); } if (timers.isEmpty()) { break; } reduceFnRunner.onTimers(timers); timers.clear(); } }
/** * Get timer data. * @param timerInternals in-memory timer internals. * @return list of timer datas. */ private List<TimerInternals.TimerData> getEligibleTimers(final InMemoryTimerInternals timerInternals) { final List<TimerInternals.TimerData> timerData = new LinkedList<>(); while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; timerData.add(timer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; timerData.add(timer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; timerData.add(timer); } if (!hasFired) { break; } } return timerData; }
/** * Advance the input watermark to the specified time, then advance the output watermark as far as * possible. */ public void advanceInputWatermark(Instant newInputWatermark) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); while (timerInternals.removeNextEventTimer() != null) { // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 } }
@Override public boolean hasNext() { // Advance if (!hasAdvance) { try { // Finish any pending windows by advancing the input watermark to infinity. timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); // Finally, advance the processing time to infinity to fire any timers. timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); timerInternals.advanceSynchronizedProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); } catch (Exception e) { throw new RuntimeException(e); } hasAdvance = true; } // Get timer data return (timerData = timerInternals.removeNextEventTimer()) != null || (timerData = timerInternals.removeNextProcessingTimer()) != null || (timerData = timerInternals.removeNextSynchronizedProcessingTimer()) != null; }
private void fireEligibleTimers( InMemoryTimerInternals timerInternals, DoFnRunner<KV<K, V>, OutputT> runner) throws Exception { while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } if (!hasFired) { break; } } }
private void fireEligibleTimers( InMemoryTimerInternals timerInternals, DoFnRunner<KV<K, V>, OutputT> runner) throws Exception { while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } if (!hasFired) { break; } } }
private void fireEligibleTimers( InMemoryTimerInternals timerInternals, DoFnRunner<KV<K, V>, OutputT> runner) throws Exception { while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } if (!hasFired) { break; } } }
/** * Fires all timers which are ready to be fired. This is done in a loop because timers may itself * schedule timers. */ private void fireEligibleTimers( InMemoryTimerInternals timerInternals, BiConsumer<String, WindowedValue> timerConsumer) { boolean hasFired; do { hasFired = false; TimerInternals.TimerData timer; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } } while (hasFired); }
/** * Fires all timers which are ready to be fired. This is done in a loop because timers may itself * schedule timers. */ private void fireEligibleTimers( InMemoryTimerInternals timerInternals, BiConsumer<String, WindowedValue> timerConsumer) { boolean hasFired; do { hasFired = false; TimerInternals.TimerData timer; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } } while (hasFired); }
@Test public void testFiringEventTimers() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTimer1 = TimerData.of(ID1, NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData eventTimer2 = TimerData.of(ID2, NS1, new Instant(29), TimeDomain.EVENT_TIME); underTest.setTimer(eventTimer1); underTest.setTimer(eventTimer2); underTest.advanceInputWatermark(new Instant(20)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer1)); assertThat(underTest.removeNextEventTimer(), nullValue()); // Advancing just a little shouldn't refire underTest.advanceInputWatermark(new Instant(21)); assertThat(underTest.removeNextEventTimer(), nullValue()); // Adding the timer and advancing a little should refire underTest.setTimer(eventTimer1); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer1)); assertThat(underTest.removeNextEventTimer(), nullValue()); // And advancing the rest of the way should still have the other timer underTest.advanceInputWatermark(new Instant(30)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer2)); assertThat(underTest.removeNextEventTimer(), nullValue()); }
/** * Advance the input watermark to the specified time, firing any timers that should fire. Then * advance the output watermark as far as possible. */ public void advanceInputWatermark(Instant newInputWatermark) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); while (true) { TimerData timer; List<TimerInternals.TimerData> timers = new ArrayList<>(); while ((timer = timerInternals.removeNextEventTimer()) != null) { timers.add(timer); } if (timers.isEmpty()) { break; } runner.onTimers(timers); } if (autoAdvanceOutputWatermark) { Instant hold = stateInternals.earliestWatermarkHold(); if (hold == null) { WindowTracing.trace( "TestInMemoryTimerInternals.advanceInputWatermark: no holds, " + "so output watermark = input watermark"); hold = timerInternals.currentInputWatermarkTime(); } advanceOutputWatermark(hold); } runner.persist(); }
@Test public void testResetById() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); Instant earlyTimestamp = new Instant(13); Instant laterTimestamp = new Instant(42); underTest.advanceInputWatermark(new Instant(0)); underTest.setTimer(NS1, ID1, earlyTimestamp, TimeDomain.EVENT_TIME); underTest.setTimer(NS1, ID1, laterTimestamp, TimeDomain.EVENT_TIME); underTest.advanceInputWatermark(earlyTimestamp.plus(1L)); assertThat(underTest.removeNextEventTimer(), nullValue()); underTest.advanceInputWatermark(laterTimestamp.plus(1L)); assertThat( underTest.removeNextEventTimer(), equalTo(TimerData.of(ID1, NS1, laterTimestamp, TimeDomain.EVENT_TIME))); }
@Test public void testDeduplicate() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTime = TimerData.of(NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData processingTime = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); underTest.setTimer(eventTime); underTest.setTimer(eventTime); underTest.setTimer(processingTime); underTest.setTimer(processingTime); underTest.advanceProcessingTime(new Instant(20)); underTest.advanceInputWatermark(new Instant(20)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime)); assertThat(underTest.removeNextEventTimer(), nullValue()); } }
@Test public void testDeletionById() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); Instant timestamp = new Instant(42); underTest.advanceInputWatermark(new Instant(0)); underTest.setTimer(NS1, ID1, timestamp, TimeDomain.EVENT_TIME); underTest.deleteTimer(NS1, ID1); underTest.advanceInputWatermark(new Instant(43)); assertThat(underTest.removeNextEventTimer(), nullValue()); }
private static void advanceInputWatermark( InMemoryTimerInternals timerInternals, Instant newInputWatermark, DoFnRunner<?, ?> toTrigger) throws Exception { timerInternals.advanceInputWatermark(newInputWatermark); TimerInternals.TimerData timer; while ((timer = timerInternals.removeNextEventTimer()) != null) { StateNamespace namespace = timer.getNamespace(); checkArgument(namespace instanceof StateNamespaces.WindowNamespace); BoundedWindow window = ((StateNamespaces.WindowNamespace) namespace).getWindow(); toTrigger.onTimer(timer.getTimerId(), window, timer.getTimestamp(), timer.getDomain()); } }
underTest.setTimer(synchronizedProcessingTime2); assertThat(underTest.removeNextEventTimer(), nullValue()); underTest.advanceInputWatermark(new Instant(30)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime1)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime2)); assertThat(underTest.removeNextEventTimer(), nullValue());