private void fireEligibleTimers( InMemoryTimerInternals timerInternals, ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner) throws Exception { List<TimerInternals.TimerData> timers = new ArrayList<>(); while (true) { TimerInternals.TimerData timer; while ((timer = timerInternals.removeNextEventTimer()) != null) { timers.add(timer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { timers.add(timer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { timers.add(timer); } if (timers.isEmpty()) { break; } reduceFnRunner.onTimers(timers); timers.clear(); } }
/** * Get timer data. * @param timerInternals in-memory timer internals. * @return list of timer datas. */ private List<TimerInternals.TimerData> getEligibleTimers(final InMemoryTimerInternals timerInternals) { final List<TimerInternals.TimerData> timerData = new LinkedList<>(); while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; timerData.add(timer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; timerData.add(timer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; timerData.add(timer); } if (!hasFired) { break; } } return timerData; }
@Override public boolean hasNext() { // Advance if (!hasAdvance) { try { // Finish any pending windows by advancing the input watermark to infinity. timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); // Finally, advance the processing time to infinity to fire any timers. timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); timerInternals.advanceSynchronizedProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE); } catch (Exception e) { throw new RuntimeException(e); } hasAdvance = true; } // Get timer data return (timerData = timerInternals.removeNextEventTimer()) != null || (timerData = timerInternals.removeNextProcessingTimer()) != null || (timerData = timerInternals.removeNextSynchronizedProcessingTimer()) != null; }
/** * Advances processing time by a given duration and, if any timers fired, performs a non-seed * {@link DoFn.ProcessElement} call, feeding it the timers. */ boolean advanceProcessingTimeBy(Duration duration) throws Exception { currentProcessingTime = currentProcessingTime.plus(duration); timerInternals.advanceProcessingTime(currentProcessingTime); List<TimerInternals.TimerData> timers = new ArrayList<>(); TimerInternals.TimerData nextTimer; while ((nextTimer = timerInternals.removeNextProcessingTimer()) != null) { timers.add(nextTimer); } if (timers.isEmpty()) { return false; } tester.processElement( KeyedWorkItems.timersWorkItem("key".getBytes(StandardCharsets.UTF_8), timers)); return true; }
/** Advance the processing time to the specified time, firing any timers that should fire. */ public void advanceProcessingTime(Instant newProcessingTime) throws Exception { timerInternals.advanceProcessingTime(newProcessingTime); ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); while (true) { TimerData timer; List<TimerInternals.TimerData> timers = new ArrayList<>(); while ((timer = timerInternals.removeNextProcessingTimer()) != null) { timers.add(timer); } if (timers.isEmpty()) { break; } runner.onTimers(timers); } runner.persist(); }
private void fireEligibleTimers( InMemoryTimerInternals timerInternals, DoFnRunner<KV<K, V>, OutputT> runner) throws Exception { while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } if (!hasFired) { break; } } }
private void fireEligibleTimers( InMemoryTimerInternals timerInternals, DoFnRunner<KV<K, V>, OutputT> runner) throws Exception { while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } if (!hasFired) { break; } } }
private void fireEligibleTimers( InMemoryTimerInternals timerInternals, DoFnRunner<KV<K, V>, OutputT> runner) throws Exception { while (true) { TimerInternals.TimerData timer; boolean hasFired = false; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, runner); } if (!hasFired) { break; } } }
/** * Fires all timers which are ready to be fired. This is done in a loop because timers may itself * schedule timers. */ private void fireEligibleTimers( InMemoryTimerInternals timerInternals, BiConsumer<String, WindowedValue> timerConsumer) { boolean hasFired; do { hasFired = false; TimerInternals.TimerData timer; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } } while (hasFired); }
/** Advance the processing time to the specified time. */ public void advanceProcessingTime(Instant newProcessingTime) throws Exception { timerInternals.advanceProcessingTime(newProcessingTime); while (timerInternals.removeNextProcessingTimer() != null) { // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 } timerInternals.advanceSynchronizedProcessingTime(newProcessingTime); while (timerInternals.removeNextSynchronizedProcessingTimer() != null) { // TODO: Should test timer firings: see https://issues.apache.org/jira/browse/BEAM-694 } }
/** * Fires all timers which are ready to be fired. This is done in a loop because timers may itself * schedule timers. */ private void fireEligibleTimers( InMemoryTimerInternals timerInternals, BiConsumer<String, WindowedValue> timerConsumer) { boolean hasFired; do { hasFired = false; TimerInternals.TimerData timer; while ((timer = timerInternals.removeNextEventTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } while ((timer = timerInternals.removeNextProcessingTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) { hasFired = true; fireTimer(timer, timerConsumer); } } while (hasFired); }
@Test public void testFiringProcessingTimeTimers() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData processingTime1 = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); TimerData processingTime2 = TimerData.of(NS1, new Instant(29), TimeDomain.PROCESSING_TIME); underTest.setTimer(processingTime1); underTest.setTimer(processingTime2); underTest.advanceProcessingTime(new Instant(20)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime1)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // Advancing just a little shouldn't refire underTest.advanceProcessingTime(new Instant(21)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // Adding the timer and advancing a little should fire again underTest.setTimer(processingTime1); underTest.advanceProcessingTime(new Instant(21)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime1)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // And advancing the rest of the way should still have the other timer underTest.advanceProcessingTime(new Instant(30)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime2)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); }
@Test public void testDeduplicate() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTime = TimerData.of(NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData processingTime = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); underTest.setTimer(eventTime); underTest.setTimer(eventTime); underTest.setTimer(processingTime); underTest.setTimer(processingTime); underTest.advanceProcessingTime(new Instant(20)); underTest.advanceInputWatermark(new Instant(20)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime)); assertThat(underTest.removeNextEventTimer(), nullValue()); } }
assertThat(underTest.removeNextEventTimer(), nullValue()); assertThat(underTest.removeNextProcessingTimer(), nullValue()); underTest.advanceProcessingTime(new Instant(30)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime1)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime2)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); assertThat( underTest.removeNextSynchronizedProcessingTimer(), equalTo(synchronizedProcessingTime2)); assertThat(underTest.removeNextProcessingTimer(), nullValue());