/** * Construct a {@link TimerData} for the given parameters, where the timer ID is * deterministically generated from the {@code timestamp} and {@code domain}. */ public static TimerData of(StateNamespace namespace, Instant timestamp, TimeDomain domain) { String timerId = new StringBuilder() .append(domain.ordinal()) .append(':') .append(timestamp.getMillis()) .toString(); return of(timerId, namespace, timestamp, domain); }
@Override public <OutputT> FnDataReceiver<OutputT> create(String pCollectionId) { final ProcessBundleDescriptors.TimerSpec timerSpec = timerOutputIdToSpecMap.get(pCollectionId); return receivedElement -> { WindowedValue windowedValue = (WindowedValue) receivedElement; Timer timer = Preconditions.checkNotNull( (Timer) ((KV) windowedValue.getValue()).getValue(), "Received null Timer from SDK harness: %s", receivedElement); LOG.debug("Timer received: {} {}", pCollectionId, timer); for (Object window : windowedValue.getWindows()) { StateNamespace namespace = StateNamespaces.window(windowCoder, (BoundedWindow) window); TimeDomain timeDomain = timerSpec.getTimerSpec().getTimeDomain(); String timerId = timerSpec.inputCollectionId(); TimerInternals.TimerData timerData = TimerInternals.TimerData.of(timerId, namespace, timer.getTimestamp(), timeDomain); timerDataConsumer.accept(windowedValue, timerData); } }; } }
public void fireTimer(W window, Instant timestamp, TimeDomain domain) throws Exception { ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); ArrayList<TimerData> timers = new ArrayList<>(1); timers.add( TimerData.of(StateNamespaces.window(windowFn.windowCoder(), window), timestamp, domain)); runner.onTimers(timers); runner.persist(); }
/** Tests that a call to onTimer gets delegated. */ @Test public void testOnTimerCalled() { PushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(ImmutableList.of()); String timerId = "fooTimer"; IntervalWindow window = new IntervalWindow(new Instant(4), new Instant(16)); Instant timestamp = new Instant(72); // Mocking is not easily compatible with annotation analysis, so we manually record // the method call. runner.onTimer(timerId, window, new Instant(timestamp), TimeDomain.EVENT_TIME); assertThat( underlying.firedTimers, contains( TimerData.of( timerId, StateNamespaces.window(IntervalWindow.getCoder(), window), timestamp, TimeDomain.EVENT_TIME))); }
@Test public void testSerialization() { TimerDataCoder timerDataCoder = TimerDataCoder.of(GlobalWindow.Coder.INSTANCE); TimerData timerData = TimerData.of( "arbitrary-id", StateNamespaces.global(), new Instant(0), TimeDomain.EVENT_TIME); String key = "key"; ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder); timerInternals.setContext(key, StringUtf8Coder.of(), Instant.now(), null); timerInternals.setTimer(timerData); ApexTimerInternals<String> cloned; assertNotNull("Serialization", cloned = KryoCloneUtils.cloneObject(timerInternals)); cloned.setContext(key, StringUtf8Coder.of(), Instant.now(), null); Map<?, Set<Slice>> timers = cloned.getTimerSet(TimeDomain.EVENT_TIME).getMap(); assertEquals(1, timers.size()); } }
@Test public void testDeduplicate() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTime = TimerData.of(NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData processingTime = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); underTest.setTimer(eventTime); underTest.setTimer(eventTime); underTest.setTimer(processingTime); underTest.setTimer(processingTime); underTest.advanceProcessingTime(new Instant(20)); underTest.advanceInputWatermark(new Instant(20)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); assertThat(underTest.removeNextEventTimer(), equalTo(eventTime)); assertThat(underTest.removeNextEventTimer(), nullValue()); } }
@Test public void testResetById() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); Instant earlyTimestamp = new Instant(13); Instant laterTimestamp = new Instant(42); underTest.advanceInputWatermark(new Instant(0)); underTest.setTimer(NS1, ID1, earlyTimestamp, TimeDomain.EVENT_TIME); underTest.setTimer(NS1, ID1, laterTimestamp, TimeDomain.EVENT_TIME); underTest.advanceInputWatermark(earlyTimestamp.plus(1L)); assertThat(underTest.removeNextEventTimer(), nullValue()); underTest.advanceInputWatermark(laterTimestamp.plus(1L)); assertThat( underTest.removeNextEventTimer(), equalTo(TimerData.of(ID1, NS1, laterTimestamp, TimeDomain.EVENT_TIME))); }
@Test public void testEncodeDecodeEqual() throws Exception { Iterable<TimerData> timers = ImmutableList.of( TimerData.of(StateNamespaces.global(), new Instant(500L), TimeDomain.EVENT_TIME)); Iterable<WindowedValue<Integer>> elements = ImmutableList.of( WindowedValue.valueInGlobalWindow(1), WindowedValue.valueInGlobalWindow(4), WindowedValue.valueInGlobalWindow(8)); KeyedWorkItemCoder<String, Integer> coder = KeyedWorkItemCoder.of(StringUtf8Coder.of(), VarIntCoder.of(), GlobalWindow.Coder.INSTANCE); CoderProperties.coderDecodeEncodeEqual(coder, KeyedWorkItems.workItem("foo", timers, elements)); CoderProperties.coderDecodeEncodeEqual(coder, KeyedWorkItems.elementsWorkItem("foo", elements)); CoderProperties.coderDecodeEncodeEqual(coder, KeyedWorkItems.timersWorkItem("foo", timers)); }
@Test public void testTimerDataCoder() throws Exception { CoderProperties.coderDecodeEncodeEqual( TimerDataCoder.of(GlobalWindow.Coder.INSTANCE), TimerData.of( "arbitrary-id", StateNamespaces.global(), new Instant(0), TimeDomain.EVENT_TIME)); Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder(); CoderProperties.coderDecodeEncodeEqual( TimerDataCoder.of(windowCoder), TimerData.of( "another-id", StateNamespaces.window( windowCoder, new IntervalWindow(new Instant(0), new Instant(100))), new Instant(99), TimeDomain.PROCESSING_TIME)); }
@OnTimer(TIMER_ID) public void onTimer(OnTimerContext context) { onTimerInvocations.add( TimerData.of( DoFnWithTimers.TIMER_ID, StateNamespaces.window(windowCoder, (W) context.window()), context.timestamp(), context.timeDomain())); } }
@Override public void onTimer( String timerId, BoundedWindow window, Instant timestamp, TimeDomain timeDomain) { firedTimers.add( TimerData.of( timerId, StateNamespaces.window(IntervalWindow.getCoder(), (IntervalWindow) window), timestamp, timeDomain)); }
@Test public void testCompareByTimerId() { Instant timestamp = new Instant(100); StateNamespace namespace = StateNamespaces.global(); TimerData id0Timer = TimerData.of("id0", namespace, timestamp, TimeDomain.EVENT_TIME); TimerData id1Timer = TimerData.of("id1", namespace, timestamp, TimeDomain.EVENT_TIME); assertThat(id0Timer, lessThan(id1Timer)); } }
public void fireTimers(W window, TimestampedValue<TimeDomain>... timers) throws Exception { ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); ArrayList<TimerData> timerData = new ArrayList<>(timers.length); for (TimestampedValue<TimeDomain> timer : timers) { timerData.add( TimerData.of( StateNamespaces.window(windowFn.windowCoder(), window), timer.getTimestamp(), timer.getValue())); } runner.onTimers(timerData); runner.persist(); }
@Override public void setTimer( StateNamespace namespace, String timerId, Instant target, TimeDomain timeDomain) { TimerData timerData = TimerData.of(timerId, namespace, target, timeDomain); setTimer(timerData); }
@Override public TimerData decode(InputStream inStream) throws CoderException, IOException { String timerId = STRING_CODER.decode(inStream); StateNamespace namespace = StateNamespaces.fromString(STRING_CODER.decode(inStream), windowCoder); Instant timestamp = INSTANT_CODER.decode(inStream); TimeDomain domain = TimeDomain.valueOf(STRING_CODER.decode(inStream)); return TimerData.of(timerId, namespace, timestamp, domain); }
@Override public void setTimer( StateNamespace namespace, String timerId, Instant target, TimeDomain timeDomain) { setTimer(TimerData.of(timerId, namespace, target, timeDomain)); }
@Test public void testCompareByNamespace() { Instant timestamp = new Instant(100); IntervalWindow firstWindow = new IntervalWindow(new Instant(0), timestamp); IntervalWindow secondWindow = new IntervalWindow(timestamp, new Instant(200)); Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder(); StateNamespace firstWindowNs = StateNamespaces.window(windowCoder, firstWindow); StateNamespace secondWindowNs = StateNamespaces.window(windowCoder, secondWindow); TimerData secondEventTime = TimerData.of(firstWindowNs, timestamp, TimeDomain.EVENT_TIME); TimerData thirdEventTime = TimerData.of(secondWindowNs, timestamp, TimeDomain.EVENT_TIME); assertThat(secondEventTime, lessThan(thirdEventTime)); }
@Test public void testCompareByTimestamp() { Instant firstTimestamp = new Instant(100); Instant secondTimestamp = new Instant(200); StateNamespace namespace = StateNamespaces.global(); TimerData firstTimer = TimerData.of(namespace, firstTimestamp, TimeDomain.EVENT_TIME); TimerData secondTimer = TimerData.of(namespace, secondTimestamp, TimeDomain.EVENT_TIME); assertThat(firstTimer, lessThan(secondTimer)); }
@Test public void testCompareEqual() { Instant timestamp = new Instant(100); StateNamespace namespace = StateNamespaces.global(); TimerData timer = TimerData.of("id", namespace, timestamp, TimeDomain.EVENT_TIME); assertThat( timer, comparesEqualTo(TimerData.of("id", namespace, timestamp, TimeDomain.EVENT_TIME))); }
@Test public void testCompareByDomain() { Instant timestamp = new Instant(100); StateNamespace namespace = StateNamespaces.global(); TimerData eventTimer = TimerData.of(namespace, timestamp, TimeDomain.EVENT_TIME); TimerData procTimer = TimerData.of(namespace, timestamp, TimeDomain.PROCESSING_TIME); TimerData synchronizedProcTimer = TimerData.of(namespace, timestamp, TimeDomain.SYNCHRONIZED_PROCESSING_TIME); assertThat(eventTimer, lessThan(procTimer)); assertThat(eventTimer, lessThan(synchronizedProcTimer)); assertThat(procTimer, lessThan(synchronizedProcTimer)); }