@Override public void processElement(final WindowedValue<InputT> elem) { try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(metricsContainer())) { delegate.processElement(elem); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public void processElement(final WindowedValue<InputT> elem) { try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) { delegate.processElement(elem); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public void processElement(final WindowedValue<InputT> elem) { try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) { delegate.processElement(elem); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public void processElement(final WindowedValue<InputT> elem) { try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(container.getMetricsContainer(stepName))) { delegate.processElement(elem); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public Iterable<WindowedValue<KeyedWorkItem<byte[], KV<InputT, RestrictionT>>>> processElementInReadyWindows( WindowedValue<KeyedWorkItem<byte[], KV<InputT, RestrictionT>>> windowedKWI) { checkTrivialOuterWindows(windowedKWI); BoundedWindow window = getUnderlyingWindow(windowedKWI.getValue()); if (!isReady(window)) { return Collections.singletonList(windowedKWI); } underlying.processElement(windowedKWI); return Collections.emptyList(); }
@Override public final void processElement(StreamRecord<WindowedValue<InputT>> streamRecord) { checkInvokeStartBundle(); doFnRunner.processElement(streamRecord.getValue()); checkInvokeFinishBundleByCount(); }
@Override public void fireTimer(InternalTimer<?, TimerData> timer) { doFnRunner.processElement(WindowedValue.valueInGlobalWindow( KeyedWorkItems.<K, InputT>timersWorkItem( (K) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
@Override public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) { doFnRunner.processElement(WindowedValue.valueInGlobalWindow( KeyedWorkItems.<String, KV<InputT, RestrictionT>>timersWorkItem( (String) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
@Override public void onData(final WindowedValue<InputT> data) { // Do not need any push-back logic. checkAndInvokeBundle(); getDoFnRunner().processElement(data); checkAndFinishBundle(); }
@Override public void fireTimer(InternalTimer<?, TimerData> timer) { doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (K) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); } }
@Override public final void processElement( StreamRecord<WindowedValue<InputT>> streamRecord) throws Exception { checkInvokeStartBundle(); doFnRunner.processElement(streamRecord.getValue()); checkInvokeFinishBundleByCount(); }
@Override public final void processElement(StreamRecord<WindowedValue<InputT>> streamRecord) { checkInvokeStartBundle(); doFnRunner.processElement(streamRecord.getValue()); checkInvokeFinishBundleByCount(); }
/** * Emits all pushed-back data. This should be used once we know that there will not be any future * side input, i.e. that there is no point in waiting. */ private void emitAllPushedBackData() throws Exception { Iterator<WindowedValue<InputT>> it = pushedBackElementsHandler.getElements().iterator(); while (it.hasNext()) { WindowedValue<InputT> element = it.next(); // we need to set the correct key in case the operator is // a (keyed) window operator setKeyContextElement1(new StreamRecord<>(element)); doFnRunner.processElement(element); } pushedBackElementsHandler.clear(); setPushedBackWatermark(Long.MAX_VALUE); }
/** * Emits all pushed-back data. This should be used once we know that there will not be any future * side input, i.e. that there is no point in waiting. */ private void emitAllPushedBackData() throws Exception { Iterator<WindowedValue<InputT>> it = pushedBackElementsHandler.getElements().iterator(); while (it.hasNext()) { WindowedValue<InputT> element = it.next(); // we need to set the correct key in case the operator is // a (keyed) window operator setKeyContextElement1(new StreamRecord<>(element)); doFnRunner.processElement(element); } pushedBackElementsHandler.clear(); setPushedBackWatermark(Long.MAX_VALUE); }
@Override public void fireTimer(InternalTimer<?, TimerData> timer) { timerInternals.cleanupPendingTimer(timer.getNamespace()); doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (K) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); } }
@Override public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) { if (timer.getNamespace().getDomain().equals(TimeDomain.EVENT_TIME)) { // ignore this, it can only be a state cleanup timers from StatefulDoFnRunner and ProcessFn // does its own state cleanup and should never set event-time timers. return; } doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (byte[]) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
@Override public void processElement(WindowedValue<InputT> input) { // StatefulDoFnRunner always observes windows, so we need to explode for (WindowedValue<InputT> value : input.explodeWindows()) { BoundedWindow window = value.getWindows().iterator().next(); if (isLate(window)) { // The element is too late for this window. droppedDueToLateness.inc(); WindowTracing.debug( "StatefulDoFnRunner.processElement: Dropping element at {}; window:{} " + "since too far behind inputWatermark:{}", input.getTimestamp(), window, cleanupTimer.currentInputWatermarkTime()); } else { cleanupTimer.setForWindow(window); doFnRunner.processElement(value); } } }
@Override public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) { timerInternals.cleanupPendingTimer(timer.getNamespace()); if (timer.getNamespace().getDomain().equals(TimeDomain.EVENT_TIME)) { // ignore this, it can only be a state cleanup timers from StatefulDoFnRunner and ProcessFn // does its own state cleanup and should never set event-time timers. return; } doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (byte[]) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
@Override public void processElement(WindowedValue<KeyedWorkItem<K, InputT>> elem) { Iterable<WindowedValue<InputT>> nonLateElements = lateDataFilter.filter(elem.getValue().key(), elem.getValue().elementsIterable()); KeyedWorkItem<K, InputT> keyedWorkItem = KeyedWorkItems.workItem( elem.getValue().key(), elem.getValue().timersIterable(), nonLateElements); doFnRunner.processElement(elem.withValue(keyedWorkItem)); }
@Test public void testProcessElementExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.processElement(WindowedValue.valueInGlobalWindow("anyValue")); }