@Test public void testDeleteTimer() { TimerDataCoder timerDataCoder = TimerDataCoder.of(GlobalWindow.Coder.INSTANCE); String key1 = "key1"; Instant instant0 = new Instant(0); Instant instant1 = new Instant(1); ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder); timerInternals.setContext(key1, StringUtf8Coder.of(), Instant.now(), null); TimerData timerData0 = TimerData.of("timerData0", StateNamespaces.global(), instant0, TimeDomain.EVENT_TIME); timerInternals.setTimer(timerData0); TimerData timerData1 = TimerData.of("timerData1", StateNamespaces.global(), instant1, TimeDomain.EVENT_TIME); timerInternals.setTimer(timerData1); Map<?, Set<Slice>> timerMap = timerInternals.getTimerSet(TimeDomain.EVENT_TIME).getMap(); assertEquals(1, timerMap.size()); assertEquals(2, timerMap.values().iterator().next().size()); timerInternals.deleteTimer(timerData0.getNamespace(), timerData0.getTimerId()); assertEquals(1, timerMap.size()); assertEquals(1, timerMap.values().iterator().next().size()); timerInternals.deleteTimer( timerData1.getNamespace(), timerData1.getTimerId(), TimeDomain.PROCESSING_TIME); assertEquals(1, timerMap.size()); assertEquals(1, timerMap.values().iterator().next().size()); timerInternals.deleteTimer( timerData1.getNamespace(), timerData1.getTimerId(), TimeDomain.EVENT_TIME); assertEquals(0, timerMap.size()); }
/** @deprecated use {@link #setTimer(StateNamespace, String, Instant, TimeDomain)}. */ @Deprecated @Override public void setTimer(TimerData timerData) { WindowTracing.trace("{}.setTimer: {}", getClass().getSimpleName(), timerData); @Nullable TimerData existing = existingTimers.get(timerData.getNamespace(), timerData.getTimerId()); if (existing == null) { existingTimers.put(timerData.getNamespace(), timerData.getTimerId(), timerData); timersForDomain(timerData.getDomain()).add(timerData); } else { checkArgument( timerData.getDomain().equals(existing.getDomain()), "Attempt to set %s for time domain %s, but it is already set for time domain %s", timerData.getTimerId(), timerData.getDomain(), existing.getDomain()); if (!timerData.getTimestamp().equals(existing.getTimestamp())) { NavigableSet<TimerData> timers = timersForDomain(timerData.getDomain()); timers.remove(existing); timers.add(timerData); existingTimers.put(timerData.getNamespace(), timerData.getTimerId(), timerData); } } }
timer.getNamespace() instanceof WindowNamespace, "Expected timer to be in WindowNamespace, but was in %s", timer.getNamespace()); @SuppressWarnings("unchecked") WindowNamespace<W> windowNamespace = (WindowNamespace<W>) timer.getNamespace(); W window = windowNamespace.getWindow(); if (TimeDomain.EVENT_TIME != timer.getDomain() && windowIsExpired(window)) { continue;
(Coder<BoundedWindow>) inputWindowingStrategy.getWindowFn().windowCoder(), window); } else { stateNamespace = timer.getNamespace(); TimerInternals.TimerData.of(stateNamespace, wakeupTime, TimeDomain.PROCESSING_TIME));
TimerData timerData = CoderUtils.decodeFromByteArray(timers.timerDataCoder, timerIt.next().buffer); if (timerData.getTimestamp().isBefore(currentTime)) { toFire.put(keyWithTimers.getKey(), timerData); timerIt.remove();
StateNamespace namespace = StateNamespaces.window(windowCoder, (BoundedWindow) window); TimerInternals.TimerData timerData = TimerInternals.TimerData.of( timerSpec.inputCollectionId(), namespace,
@Test public void testTimerOrdering() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTime1 = TimerData.of(NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData processingTime1 = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); TimerData synchronizedProcessingTime1 = TimerData.of(NS1, new Instant(19), TimeDomain.SYNCHRONIZED_PROCESSING_TIME); TimerData eventTime2 = TimerData.of(NS1, new Instant(29), TimeDomain.EVENT_TIME); TimerData processingTime2 = TimerData.of(NS1, new Instant(29), TimeDomain.PROCESSING_TIME); TimerData synchronizedProcessingTime2 = TimerData.of(NS1, new Instant(29), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
StateNamespace namespace = StateNamespaces.window(windowCoder, (BoundedWindow) window); TimerInternals.TimerData timerData = TimerInternals.TimerData.of( timerSpec.inputCollectionId(), namespace,
fn.onTimerInvocations, contains( TimerData.of( DoFnWithTimers.TIMER_ID, StateNamespaces.window(windowFn.windowCoder(), GlobalWindow.INSTANCE),
public void deleteTimer(Slice keyBytes, StateNamespace namespace, String timerId) { Set<Slice> timersForKey = activeTimers.get(keyBytes); if (timersForKey == null) { return; } Iterator<Slice> timerIt = timersForKey.iterator(); while (timerIt.hasNext()) { try { TimerData timerData = CoderUtils.decodeFromByteArray(timerDataCoder, timerIt.next().buffer); ComparisonChain chain = ComparisonChain.start().compare(timerData.getTimerId(), timerId); if (chain.result() == 0 && !timerData.getNamespace().equals(namespace)) { // Obtaining the stringKey may be expensive; only do so if required chain = chain.compare(timerData.getNamespace().stringKey(), namespace.stringKey()); } if (chain.result() == 0) { timerIt.remove(); } } catch (CoderException e) { throw new RuntimeException(e); } } if (timersForKey.isEmpty()) { activeTimers.remove(keyBytes); } }
@Test public void testFiringProcessingTimeTimers() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData processingTime1 = TimerData.of(NS1, new Instant(19), TimeDomain.PROCESSING_TIME); TimerData processingTime2 = TimerData.of(NS1, new Instant(29), TimeDomain.PROCESSING_TIME); underTest.setTimer(processingTime1); underTest.setTimer(processingTime2); underTest.advanceProcessingTime(new Instant(20)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime1)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // Advancing just a little shouldn't refire underTest.advanceProcessingTime(new Instant(21)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // Adding the timer and advancing a little should fire again underTest.setTimer(processingTime1); underTest.advanceProcessingTime(new Instant(21)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime1)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); // And advancing the rest of the way should still have the other timer underTest.advanceProcessingTime(new Instant(30)); assertThat(underTest.removeNextProcessingTimer(), equalTo(processingTime2)); assertThat(underTest.removeNextProcessingTimer(), nullValue()); }
@Override public <OutputT> FnDataReceiver<OutputT> create(String pCollectionId) { final ProcessBundleDescriptors.TimerSpec timerSpec = timerOutputIdToSpecMap.get(pCollectionId); return receivedElement -> { WindowedValue windowedValue = (WindowedValue) receivedElement; Timer timer = Preconditions.checkNotNull( (Timer) ((KV) windowedValue.getValue()).getValue(), "Received null Timer from SDK harness: %s", receivedElement); LOG.debug("Timer received: {} {}", pCollectionId, timer); for (Object window : windowedValue.getWindows()) { StateNamespace namespace = StateNamespaces.window(windowCoder, (BoundedWindow) window); TimeDomain timeDomain = timerSpec.getTimerSpec().getTimeDomain(); String timerId = timerSpec.inputCollectionId(); TimerInternals.TimerData timerData = TimerInternals.TimerData.of(timerId, namespace, timer.getTimestamp(), timeDomain); timerDataConsumer.accept(windowedValue, timerData); } }; } }
@Override public <OutputT> FnDataReceiver<OutputT> create(String pCollectionId) { final ProcessBundleDescriptors.TimerSpec timerSpec = timerOutputIdToSpecMap.get(pCollectionId); return receivedElement -> { WindowedValue windowedValue = (WindowedValue) receivedElement; Timer timer = Preconditions.checkNotNull( (Timer) ((KV) windowedValue.getValue()).getValue(), "Received null Timer from SDK harness: %s", receivedElement); LOG.debug("Timer received: {} {}", pCollectionId, timer); for (Object window : windowedValue.getWindows()) { StateNamespace namespace = StateNamespaces.window(windowCoder, (BoundedWindow) window); TimeDomain timeDomain = timerSpec.getTimerSpec().getTimeDomain(); String timerId = timerSpec.inputCollectionId(); TimerInternals.TimerData timerData = TimerInternals.TimerData.of(timerId, namespace, timer.getTimestamp(), timeDomain); timerDataConsumer.accept(windowedValue, timerData); } }; } }
/** * {@inheritDoc}. * * <p>Used for sorting {@link TimerData} by timestamp. Furthermore, we compare timers by all the * other fields so that {@code compareTo()} only returns 0 when {@code equals()} returns 0. This * ensures consistent sort order. */ @Override public int compareTo(TimerData that) { if (this.equals(that)) { return 0; } ComparisonChain chain = ComparisonChain.start() .compare(this.getTimestamp(), that.getTimestamp()) .compare(this.getDomain(), that.getDomain()) .compare(this.getTimerId(), that.getTimerId()); if (chain.result() == 0 && !this.getNamespace().equals(that.getNamespace())) { // Obtaining the stringKey may be expensive; only do so if required chain = chain.compare(getNamespace().stringKey(), that.getNamespace().stringKey()); } return chain.result(); } }
@Test public void testFiringEventTimers() throws Exception { InMemoryTimerInternals underTest = new InMemoryTimerInternals(); TimerData eventTimer1 = TimerData.of(ID1, NS1, new Instant(19), TimeDomain.EVENT_TIME); TimerData eventTimer2 = TimerData.of(ID2, NS1, new Instant(29), TimeDomain.EVENT_TIME); underTest.setTimer(eventTimer1); underTest.setTimer(eventTimer2); underTest.advanceInputWatermark(new Instant(20)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer1)); assertThat(underTest.removeNextEventTimer(), nullValue()); // Advancing just a little shouldn't refire underTest.advanceInputWatermark(new Instant(21)); assertThat(underTest.removeNextEventTimer(), nullValue()); // Adding the timer and advancing a little should refire underTest.setTimer(eventTimer1); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer1)); assertThat(underTest.removeNextEventTimer(), nullValue()); // And advancing the rest of the way should still have the other timer underTest.advanceInputWatermark(new Instant(30)); assertThat(underTest.removeNextEventTimer(), equalTo(eventTimer2)); assertThat(underTest.removeNextEventTimer(), nullValue()); }
/** Tests that a call to onTimer gets delegated. */ @Test public void testOnTimerCalled() { PushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(ImmutableList.of()); String timerId = "fooTimer"; IntervalWindow window = new IntervalWindow(new Instant(4), new Instant(16)); Instant timestamp = new Instant(72); // Mocking is not easily compatible with annotation analysis, so we manually record // the method call. runner.onTimer(timerId, window, new Instant(timestamp), TimeDomain.EVENT_TIME); assertThat( underlying.firedTimers, contains( TimerData.of( timerId, StateNamespaces.window(IntervalWindow.getCoder(), window), timestamp, TimeDomain.EVENT_TIME))); }
/** * Returns when the next timer in the given time domain will fire, or {@code null} if there are no * timers scheduled in that time domain. */ @Nullable public Instant getNextTimer(TimeDomain domain) { try { switch (domain) { case EVENT_TIME: return watermarkTimers.first().getTimestamp(); case PROCESSING_TIME: return processingTimers.first().getTimestamp(); case SYNCHRONIZED_PROCESSING_TIME: return synchronizedProcessingTimers.first().getTimestamp(); default: throw new IllegalArgumentException("Unexpected time domain: " + domain); } } catch (NoSuchElementException exc) { return null; } }
@Test public void testSerialization() { TimerDataCoder timerDataCoder = TimerDataCoder.of(GlobalWindow.Coder.INSTANCE); TimerData timerData = TimerData.of( "arbitrary-id", StateNamespaces.global(), new Instant(0), TimeDomain.EVENT_TIME); String key = "key"; ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder); timerInternals.setContext(key, StringUtf8Coder.of(), Instant.now(), null); timerInternals.setTimer(timerData); ApexTimerInternals<String> cloned; assertNotNull("Serialization", cloned = KryoCloneUtils.cloneObject(timerInternals)); cloned.setContext(key, StringUtf8Coder.of(), Instant.now(), null); Map<?, Set<Slice>> timers = cloned.getTimerSet(TimeDomain.EVENT_TIME).getMap(); assertEquals(1, timers.size()); } }
@Override public void fireTimer(Object key, Collection<TimerData> timerDataSet) { pushbackDoFnRunner.startBundle(); @SuppressWarnings("unchecked") Coder<Object> keyCoder = (Coder) currentKeyStateInternals.getKeyCoder(); ((StateInternalsProxy) currentKeyStateInternals).setKey(key); currentKeyTimerInternals.setContext( key, keyCoder, new Instant(this.currentInputWatermark), new Instant(this.currentOutputWatermark)); for (TimerData timerData : timerDataSet) { StateNamespace namespace = timerData.getNamespace(); checkArgument(namespace instanceof WindowNamespace); BoundedWindow window = ((WindowNamespace<?>) namespace).getWindow(); pushbackDoFnRunner.onTimer( timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain()); } pushbackDoFnRunner.finishBundle(); }