@Override public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) { doFnRunner.processElement(WindowedValue.valueInGlobalWindow( KeyedWorkItems.<String, KV<InputT, RestrictionT>>timersWorkItem( (String) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
/** * Process the collected data and trigger timers. * @param inputWatermark current input watermark * @param processingTime processing time * @param synchronizedTime synchronized time */ private void processElementsAndTriggerTimers(final Watermark inputWatermark, final Instant processingTime, final Instant synchronizedTime) { for (final Map.Entry<K, List<WindowedValue<InputT>>> entry : keyToValues.entrySet()) { final K key = entry.getKey(); final List<WindowedValue<InputT>> values = entry.getValue(); // for each key // Process elements if (!values.isEmpty()) { final KeyedWorkItem<K, InputT> keyedWorkItem = KeyedWorkItems.elementsWorkItem(key, values); // The DoFnRunner interface requires WindowedValue, // but this windowed value is actually not used in the ReduceFnRunner internal. getDoFnRunner().processElement(WindowedValue.valueInGlobalWindow(keyedWorkItem)); } // Trigger timers triggerTimers(key, inputWatermark, processingTime, synchronizedTime); // Remove values values.clear(); } }
@Override public KeyedWorkItem<K, ElemT> decode(InputStream inStream) throws CoderException, IOException { K key = keyCoder.decode(inStream); Iterable<TimerData> timers = timersCoder.decode(inStream); Iterable<WindowedValue<ElemT>> elems = elemsCoder.decode(inStream); return KeyedWorkItems.workItem(key, timers, elems); }
@Test public void testEncodeDecodeEqual() throws Exception { Iterable<TimerData> timers = ImmutableList.of( TimerData.of(StateNamespaces.global(), new Instant(500L), TimeDomain.EVENT_TIME)); Iterable<WindowedValue<Integer>> elements = ImmutableList.of( WindowedValue.valueInGlobalWindow(1), WindowedValue.valueInGlobalWindow(4), WindowedValue.valueInGlobalWindow(8)); KeyedWorkItemCoder<String, Integer> coder = KeyedWorkItemCoder.of(StringUtf8Coder.of(), VarIntCoder.of(), GlobalWindow.Coder.INSTANCE); CoderProperties.coderDecodeEncodeEqual(coder, KeyedWorkItems.workItem("foo", timers, elements)); CoderProperties.coderDecodeEncodeEqual(coder, KeyedWorkItems.elementsWorkItem("foo", elements)); CoderProperties.coderDecodeEncodeEqual(coder, KeyedWorkItems.timersWorkItem("foo", timers)); }
@Override public void fireTimer(InternalTimer<?, TimerData> timer) { doFnRunner.processElement(WindowedValue.valueInGlobalWindow( KeyedWorkItems.<K, InputT>timersWorkItem( (K) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
void startElement(WindowedValue<KV<InputT, RestrictionT>> windowedValue) throws Exception { tester.processElement( KeyedWorkItems.elementsWorkItem( "key".getBytes(StandardCharsets.UTF_8), Collections.singletonList(windowedValue))); }
@Override public void processElement(WindowedValue<KeyedWorkItem<K, InputT>> elem) { Iterable<WindowedValue<InputT>> nonLateElements = lateDataFilter.filter(elem.getValue().key(), elem.getValue().elementsIterable()); KeyedWorkItem<K, InputT> keyedWorkItem = KeyedWorkItems.workItem( elem.getValue().key(), elem.getValue().timersIterable(), nonLateElements); doFnRunner.processElement(elem.withValue(keyedWorkItem)); }
@Override public void fireTimer(InternalTimer<?, TimerData> timer) { doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (K) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); } }
@Override public final void process( ApexStreamTuple<WindowedValue<KV<K, V>>> tuple, OutputEmitter<ApexStreamTuple<? extends WindowedValue<?>>> outputEmitter) { if (tuple instanceof ApexStreamTuple.WatermarkTuple) { outputEmitter.emit(tuple); } else { for (WindowedValue<KV<K, V>> in : tuple.getValue().explodeWindows()) { KeyedWorkItem<K, V> kwi = KeyedWorkItems.elementsWorkItem( in.getValue().getKey(), Collections.singletonList(in.withValue(in.getValue().getValue()))); outputEmitter.emit(ApexStreamTuple.DataTuple.of(in.withValue(kwi))); } } } }
@Override public void fireTimer(InternalTimer<?, TimerData> timer) { timerInternals.cleanupPendingTimer(timer.getNamespace()); doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (K) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); } }
@Override public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) { if (timer.getNamespace().getDomain().equals(TimeDomain.EVENT_TIME)) { // ignore this, it can only be a state cleanup timers from StatefulDoFnRunner and ProcessFn // does its own state cleanup and should never set event-time timers. return; } doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (byte[]) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
@Override public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) { timerInternals.cleanupPendingTimer(timer.getNamespace()); if (timer.getNamespace().getDomain().equals(TimeDomain.EVENT_TIME)) { // ignore this, it can only be a state cleanup timers from StatefulDoFnRunner and ProcessFn // does its own state cleanup and should never set event-time timers. return; } doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (byte[]) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
/** * Advances processing time by a given duration and, if any timers fired, performs a non-seed * {@link DoFn.ProcessElement} call, feeding it the timers. */ boolean advanceProcessingTimeBy(Duration duration) throws Exception { currentProcessingTime = currentProcessingTime.plus(duration); timerInternals.advanceProcessingTime(currentProcessingTime); List<TimerInternals.TimerData> timers = new ArrayList<>(); TimerInternals.TimerData nextTimer; while ((nextTimer = timerInternals.removeNextProcessingTimer()) != null) { timers.add(nextTimer); } if (timers.isEmpty()) { return false; } tester.processElement( KeyedWorkItems.timersWorkItem("key".getBytes(StandardCharsets.UTF_8), timers)); return true; }
KeyedWorkItems.timersWorkItem(key, timerDataList);